diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index 3022f94a599..24cb314cd27 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -130,37 +130,24 @@ def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, fr text: Final = _uncommented(script.read_text()) return MappingProxyType( { - label: frozenset( - match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body) - ) + label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)) for label, body in SELECTION_ARM_RE.findall(text) } ) def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]: - return frozenset( - token for tokens in _unit_selection_arms(repo_root).values() for token in tokens - ) + return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens) def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]: - return frozenset( - scalar.value - for scalar in scalars - if scalar.key == "unit-flag" and "${{" not in scalar.value - ) + return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value) -def _shard_tokens( - scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]] -) -> frozenset[str]: +def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]: wired: Final = _wired_unit_flags(scalars) return _invoked_test_tokens(scalars) | frozenset( - token - for label, tokens in arms.items() - if label in wired - for token in tokens + token for label, tokens in arms.items() if label in wired for token in tokens ) @@ -544,17 +531,37 @@ def _integration_groups(runner: pathlib.Path) -> dict[str, tuple[str, ...]]: return {group: tuple(folders) for group, folders in ast.literal_eval(mapping).items()} +def _integration_github_files(runner: pathlib.Path) -> frozenset[str]: + module: Final = ast.parse(runner.read_text()) + literal: Final = next( + ( + node.value + for node in module.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "GITHUB_FILES" + ), + None, + ) + if literal is None: + return frozenset() + values: Final = literal.args[0] if isinstance(literal, ast.Call) else literal + return frozenset(ast.literal_eval(values)) + + def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]: runner: Final = repo_root / "tests/integration/run.py" if not runner.exists(): return frozenset(), () groups: Final = _integration_groups(runner) + github_files: Final = _integration_github_files(runner) integration_root: Final = repo_root / "tests/integration" paths: Final = frozenset( str(path.relative_to(repo_root)) for folders in groups.values() for folder in folders for path in (integration_root / folder).rglob("test_*.py") + if str(path.relative_to(repo_root)) not in github_files ) browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json" browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else () @@ -595,10 +602,22 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens for path in (repo_root / ".github/workflows").glob("*.y*ml") for scalar in _scalars(yaml.safe_load(path.read_text()), path.name) ) - findings: Final = tuple( - Finding(path, "integration contract is also selected by GitHub Actions") - for path in paths - if any(_token_covers(token, path) for token in gha_tokens) + findings: Final = ( + tuple( + Finding(path, "integration contract is also selected by GitHub Actions") + for path in paths + if any(_token_covers(token, path) for token in gha_tokens) + ) + + tuple( + Finding(path, "GitHub-owned integration contract has no invoking workflow") + for path in sorted(github_files) + if not any(_token_covers(token, path) for token in gha_tokens) + ) + + tuple( + Finding(path, "GitHub-owned integration file is missing") + for path in sorted(github_files) + if not (repo_root / path).is_file() + ) ) browser_commands: Final = tuple( scalar.value @@ -642,7 +661,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens return frozenset(), findings + ( Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"), ) - return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings + return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings def main() -> int: diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml index 519d387976e..a1c639acbeb 100644 --- a/.github/workflows/test-postgres.yml +++ b/.github/workflows/test-postgres.yml @@ -45,6 +45,13 @@ jobs: fail-fast: false matrix: include: + - shard: roi-database + test-path: "tests/integration/database/test_roi_observed.py" + seed: none + workers: 0 + timeout-minutes: 10 + job-timeout-minutes: 35 + - shard: proxy-behavior test-path: "tests/proxy_behavior" seed: db-push @@ -147,7 +154,7 @@ jobs: env: TEST_PATH: ${{ matrix.test-path }} WORKERS: ${{ matrix.workers }} - PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }} + PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || matrix.shard == 'roi-database' && '--cov=./litellm --cov-report=xml:coverage-roi-postgres.xml' || '' }} run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 @@ -165,3 +172,14 @@ jobs: files: coverage-lens-postgres.xml flags: lens-postgres fail_ci_if_error: true + + - name: Upload ROI database coverage + if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'roi-database' && !cancelled() + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + use_oidc: true + version: v11.3.1 + root_dir: ${{ github.workspace }} + files: coverage-roi-postgres.xml + flags: roi-postgres + fail_ci_if_error: true diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4cf6db6f257..7b9af2f2b98 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -49014,6 +49014,963 @@ "title": "HTTPValidationError", "type": "object" }, + "ObservedAccount": { + "properties": { + "connection_id": { + "title": "Connection Id", + "type": "string" + }, + "login": { + "title": "Login", + "type": "string" + } + }, + "required": [ + "connection_id", + "login" + ], + "title": "ObservedAccount", + "type": "object" + }, + "ObservedApp": { + "properties": { + "api_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Api Url" + }, + "callback_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Callback Url" + }, + "can_install": { + "default": false, + "title": "Can Install", + "type": "boolean" + }, + "configured": { + "title": "Configured", + "type": "boolean" + } + }, + "required": [ + "configured" + ], + "title": "ObservedApp", + "type": "object" + }, + "ObservedApps": { + "properties": { + "github": { + "$ref": "#/components/schemas/ObservedApp" + }, + "gitlab": { + "$ref": "#/components/schemas/ObservedApp" + } + }, + "required": [ + "github", + "gitlab" + ], + "title": "ObservedApps", + "type": "object" + }, + "ObservedAuthorization": { + "properties": { + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "ObservedAuthorization", + "type": "object" + }, + "ObservedConnection": { + "properties": { + "api_url": { + "title": "Api Url", + "type": "string" + }, + "connection_type": { + "enum": [ + "token", + "app" + ], + "title": "Connection Type", + "type": "string" + }, + "has_token": { + "title": "Has Token", + "type": "boolean" + }, + "id": { + "default": "", + "title": "Id", + "type": "string" + }, + "ready": { + "title": "Ready", + "type": "boolean" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, + "update_interval_minutes": { + "title": "Update Interval Minutes", + "type": "number" + } + }, + "required": [ + "source_provider", + "api_url", + "repos", + "has_token", + "update_interval_minutes", + "ready", + "connection_type" + ], + "title": "ObservedConnection", + "type": "object" + }, + "ObservedConnectionIdentities": { + "properties": { + "api_url": { + "title": "Api Url", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "identity_map": { + "additionalProperties": { + "type": "string" + }, + "title": "Identity Map", + "type": "object" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, + "unmatched_logins": { + "items": { + "type": "string" + }, + "title": "Unmatched Logins", + "type": "array" + } + }, + "required": [ + "id", + "source_provider", + "api_url", + "repos", + "identity_map", + "unmatched_logins" + ], + "title": "ObservedConnectionIdentities", + "type": "object" + }, + "ObservedHumanSummary": { + "properties": { + "median_merge_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Median Merge Hours" + } + }, + "required": [ + "median_merge_hours" + ], + "title": "ObservedHumanSummary", + "type": "object" + }, + "ObservedIdentities": { + "properties": { + "connections": { + "default": [], + "items": { + "$ref": "#/components/schemas/ObservedConnectionIdentities" + }, + "title": "Connections", + "type": "array" + }, + "gateway_emails": { + "items": { + "type": "string" + }, + "title": "Gateway Emails", + "type": "array" + }, + "identity_map": { + "additionalProperties": { + "type": "string" + }, + "title": "Identity Map", + "type": "object" + }, + "unmatched_logins": { + "items": { + "type": "string" + }, + "title": "Unmatched Logins", + "type": "array" + } + }, + "required": [ + "gateway_emails", + "identity_map", + "unmatched_logins" + ], + "title": "ObservedIdentities", + "type": "object" + }, + "ObservedIdentityUpdate": { + "additionalProperties": false, + "properties": { + "accounts": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/ObservedAccount" + }, + "maxItems": 500, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Accounts" + }, + "email": { + "title": "Email", + "type": "string" + }, + "logins": { + "default": [], + "items": { + "type": "string" + }, + "maxItems": 100, + "title": "Logins", + "type": "array" + } + }, + "required": [ + "email" + ], + "title": "ObservedIdentityUpdate", + "type": "object" + }, + "ObservedPeriod": { + "properties": { + "agent_authored": { + "title": "Agent Authored", + "type": "integer" + }, + "agents_without_requester": { + "title": "Agents Without Requester", + "type": "integer" + }, + "explicitly_titled_revert_prs": { + "title": "Explicitly Titled Revert Prs", + "type": "integer" + }, + "human_authored": { + "title": "Human Authored", + "type": "integer" + }, + "human_summary": { + "$ref": "#/components/schemas/ObservedHumanSummary" + }, + "matched_internal_prs": { + "title": "Matched Internal Prs", + "type": "integer" + }, + "matched_users_recorded_spend": { + "title": "Matched Users Recorded Spend", + "type": "number" + }, + "median_merge_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Median Merge Hours" + }, + "merged_prs": { + "title": "Merged Prs", + "type": "integer" + }, + "missing_author": { + "title": "Missing Author", + "type": "integer" + }, + "new_bug_labeled_issues": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "New Bug Labeled Issues" + }, + "new_regression_labeled_issues": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "New Regression Labeled Issues" + }, + "spend_observation": { + "enum": [ + "records_present", + "no_records" + ], + "title": "Spend Observation", + "type": "string" + }, + "window": { + "$ref": "#/components/schemas/ObservedWindow" + } + }, + "required": [ + "window", + "merged_prs", + "median_merge_hours", + "human_authored", + "agent_authored", + "missing_author", + "agents_without_requester", + "matched_internal_prs", + "new_bug_labeled_issues", + "new_regression_labeled_issues", + "explicitly_titled_revert_prs", + "matched_users_recorded_spend", + "spend_observation", + "human_summary" + ], + "title": "ObservedPeriod", + "type": "object" + }, + "ObservedPeriods": { + "properties": { + "current": { + "$ref": "#/components/schemas/ObservedPeriod" + }, + "last_year": { + "$ref": "#/components/schemas/ObservedPeriod" + }, + "previous": { + "$ref": "#/components/schemas/ObservedPeriod" + } + }, + "required": [ + "current", + "previous", + "last_year" + ], + "title": "ObservedPeriods", + "type": "object" + }, + "ObservedPerson": { + "properties": { + "accounts": { + "default": [], + "items": { + "$ref": "#/components/schemas/ObservedAccount" + }, + "title": "Accounts", + "type": "array" + }, + "email": { + "title": "Email", + "type": "string" + }, + "logins": { + "items": { + "type": "string" + }, + "title": "Logins", + "type": "array" + }, + "name": { + "title": "Name", + "type": "string" + }, + "periods": { + "$ref": "#/components/schemas/ObservedPersonPeriods" + } + }, + "required": [ + "name", + "email", + "logins", + "periods" + ], + "title": "ObservedPerson", + "type": "object" + }, + "ObservedPersonPeriod": { + "properties": { + "declared_agent_owned": { + "title": "Declared Agent Owned", + "type": "integer" + }, + "direct_authored": { + "title": "Direct Authored", + "type": "integer" + }, + "gateway_recorded_spend": { + "title": "Gateway Recorded Spend", + "type": "number" + }, + "median_merge_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Median Merge Hours" + }, + "merged_prs": { + "title": "Merged Prs", + "type": "integer" + }, + "pr_urls": { + "items": { + "type": "string" + }, + "title": "Pr Urls", + "type": "array" + }, + "prs_per_week": { + "title": "Prs Per Week", + "type": "number" + }, + "recorded_spend_per_attributed_pr": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Recorded Spend Per Attributed Pr" + }, + "spend_observation": { + "enum": [ + "records_present", + "no_records" + ], + "title": "Spend Observation", + "type": "string" + } + }, + "required": [ + "merged_prs", + "prs_per_week", + "median_merge_hours", + "direct_authored", + "declared_agent_owned", + "gateway_recorded_spend", + "recorded_spend_per_attributed_pr", + "spend_observation", + "pr_urls" + ], + "title": "ObservedPersonPeriod", + "type": "object" + }, + "ObservedPersonPeriods": { + "properties": { + "current": { + "$ref": "#/components/schemas/ObservedPersonPeriod" + }, + "last_year": { + "$ref": "#/components/schemas/ObservedPersonPeriod" + }, + "previous": { + "$ref": "#/components/schemas/ObservedPersonPeriod" + } + }, + "required": [ + "current", + "previous", + "last_year" + ], + "title": "ObservedPersonPeriods", + "type": "object" + }, + "ObservedPullPeriods": { + "properties": { + "current": { + "items": { + "$ref": "#/components/schemas/ObservedPullResponse" + }, + "title": "Current", + "type": "array" + }, + "last_year": { + "items": { + "$ref": "#/components/schemas/ObservedPullResponse" + }, + "title": "Last Year", + "type": "array" + }, + "previous": { + "items": { + "$ref": "#/components/schemas/ObservedPullResponse" + }, + "title": "Previous", + "type": "array" + } + }, + "required": [ + "current", + "previous", + "last_year" + ], + "title": "ObservedPullPeriods", + "type": "object" + }, + "ObservedPullResponse": { + "properties": { + "agent": { + "default": false, + "title": "Agent", + "type": "boolean" + }, + "author": { + "title": "Author", + "type": "string" + }, + "branch_cost": { + "$ref": "#/components/schemas/ROIBranchAttribution" + }, + "connection_id": { + "default": "", + "title": "Connection Id", + "type": "string" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "merge_hours": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Merge Hours" + }, + "merged_at": { + "format": "date-time", + "title": "Merged At", + "type": "string" + }, + "number": { + "title": "Number", + "type": "integer" + }, + "profile_email": { + "default": "", + "title": "Profile Email", + "type": "string" + }, + "repo": { + "title": "Repo", + "type": "string" + }, + "requester": { + "default": "", + "title": "Requester", + "type": "string" + }, + "source_branch": { + "default": "", + "title": "Source Branch", + "type": "string" + }, + "source_repo": { + "default": "", + "title": "Source Repo", + "type": "string" + }, + "title": { + "title": "Title", + "type": "string" + }, + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "repo", + "number", + "title", + "url", + "author", + "merged_at", + "merge_hours", + "branch_cost" + ], + "title": "ObservedPullResponse", + "type": "object" + }, + "ObservedReport": { + "properties": { + "captured_at": { + "format": "date-time", + "title": "Captured At", + "type": "string" + }, + "connections": { + "default": [], + "items": { + "$ref": "#/components/schemas/ObservedSource" + }, + "title": "Connections", + "type": "array" + }, + "people": { + "items": { + "$ref": "#/components/schemas/ObservedPerson" + }, + "title": "People", + "type": "array" + }, + "periods": { + "$ref": "#/components/schemas/ObservedPeriods" + }, + "pulls": { + "$ref": "#/components/schemas/ObservedPullPeriods" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab", + "mixed" + ], + "title": "Source Provider", + "type": "string" + }, + "unlinked_branches": { + "items": { + "$ref": "#/components/schemas/ROIBranchSpend" + }, + "title": "Unlinked Branches", + "type": "array" + }, + "unmatched_logins": { + "items": { + "type": "string" + }, + "title": "Unmatched Logins", + "type": "array" + } + }, + "required": [ + "source_provider", + "repos", + "captured_at", + "periods", + "people", + "pulls", + "unlinked_branches", + "unmatched_logins" + ], + "title": "ObservedReport", + "type": "object" + }, + "ObservedReportResponse": { + "properties": { + "report": { + "anyOf": [ + { + "$ref": "#/components/schemas/ObservedReport" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "report" + ], + "title": "ObservedReportResponse", + "type": "object" + }, + "ObservedSettings": { + "properties": { + "api_url": { + "title": "Api Url", + "type": "string" + }, + "connection_type": { + "enum": [ + "token", + "app" + ], + "title": "Connection Type", + "type": "string" + }, + "connections": { + "default": [], + "items": { + "$ref": "#/components/schemas/ObservedConnection" + }, + "title": "Connections", + "type": "array" + }, + "has_token": { + "title": "Has Token", + "type": "boolean" + }, + "id": { + "default": "", + "title": "Id", + "type": "string" + }, + "ready": { + "title": "Ready", + "type": "boolean" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, + "update_interval_minutes": { + "title": "Update Interval Minutes", + "type": "number" + } + }, + "required": [ + "source_provider", + "api_url", + "repos", + "has_token", + "update_interval_minutes", + "ready", + "connection_type" + ], + "title": "ObservedSettings", + "type": "object" + }, + "ObservedSettingsUpdate": { + "additionalProperties": false, + "properties": { + "api_url": { + "title": "Api Url", + "type": "string" + }, + "connection_id": { + "anyOf": [ + { + "maxLength": 100, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connection Id" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" + }, + "update_interval_minutes": { + "anyOf": [ + { + "maximum": 43200.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Update Interval Minutes" + } + }, + "required": [ + "source_provider", + "api_url", + "repos" + ], + "title": "ObservedSettingsUpdate", + "type": "object" + }, + "ObservedSource": { + "properties": { + "api_url": { + "title": "Api Url", + "type": "string" + }, + "id": { + "title": "Id", + "type": "string" + }, + "repos": { + "items": { + "type": "string" + }, + "title": "Repos", + "type": "array" + }, + "source_provider": { + "enum": [ + "github", + "gitlab" + ], + "title": "Source Provider", + "type": "string" + } + }, + "required": [ + "id", + "source_provider", + "api_url", + "repos" + ], + "title": "ObservedSource", + "type": "object" + }, + "ObservedWindow": { + "properties": { + "end": { + "format": "date", + "title": "End", + "type": "string" + }, + "start": { + "format": "date", + "title": "Start", + "type": "string" + } + }, + "required": [ + "start", + "end" + ], + "title": "ObservedWindow", + "type": "object" + }, "ROIBranchAttribution": { "properties": { "branch": { @@ -49703,6 +50660,15 @@ "title": "Ready", "type": "boolean" }, + "report_mode": { + "default": "legacy", + "enum": [ + "legacy", + "observed" + ], + "title": "Report Mode", + "type": "string" + }, "repos": { "items": { "type": "string" @@ -49834,6 +50800,21 @@ ], "title": "Gitlab Token" }, + "report_mode": { + "anyOf": [ + { + "enum": [ + "legacy", + "observed" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Report Mode" + }, "repos": { "anyOf": [ { @@ -50267,6 +51248,441 @@ ] } }, + "/roi-calculator/observed/apps": { + "get": { + "operationId": "observed_apps_roi_calculator_observed_apps_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedApps" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Observed Apps", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/identities": { + "get": { + "operationId": "get_observed_identities_roi_calculator_observed_identities_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedIdentities" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Observed Identities", + "tags": [ + "roi_calculator" + ] + }, + "put": { + "operationId": "save_observed_identities_roi_calculator_observed_identities_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedIdentityUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedReportResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Save Observed Identities", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/oauth/{provider}/start": { + "post": { + "operationId": "start_observed_authorization_roi_calculator_observed_oauth__provider__start_post", + "parameters": [ + { + "in": "path", + "name": "provider", + "required": true, + "schema": { + "enum": [ + "github", + "gitlab" + ], + "title": "Provider", + "type": "string" + } + }, + { + "in": "query", + "name": "install", + "required": false, + "schema": { + "default": false, + "title": "Install", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedAuthorization" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Start Observed Authorization", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/report": { + "get": { + "operationId": "get_observed_report_roi_calculator_observed_report_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedReportResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Observed Report", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/repositories": { + "get": { + "operationId": "observed_repositories_roi_calculator_observed_repositories_get", + "parameters": [ + { + "in": "query", + "name": "connection", + "required": false, + "schema": { + "anyOf": [ + { + "maxLength": 100, + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Connection" + } + }, + { + "in": "query", + "name": "query", + "required": false, + "schema": { + "default": "", + "maxLength": 200, + "title": "Query", + "type": "string" + } + }, + { + "in": "query", + "name": "page", + "required": false, + "schema": { + "default": 1, + "maximum": 1000, + "minimum": 1, + "title": "Page", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ROIRepositoriesResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Observed Repositories", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/settings": { + "get": { + "operationId": "get_observed_settings_roi_calculator_observed_settings_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedSettings" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Observed Settings", + "tags": [ + "roi_calculator" + ] + }, + "put": { + "operationId": "save_observed_settings_roi_calculator_observed_settings_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedSettingsUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObservedSettings" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Save Observed Settings", + "tags": [ + "roi_calculator" + ] + } + }, + "/roi-calculator/observed/sync": { + "delete": { + "operationId": "cancel_observed_sync_roi_calculator_observed_sync_delete", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ROISyncStatus" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Cancel Observed Sync", + "tags": [ + "roi_calculator" + ] + }, + "get": { + "operationId": "get_observed_sync_roi_calculator_observed_sync_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ROISyncStatus" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Get Observed Sync", + "tags": [ + "roi_calculator" + ] + }, + "post": { + "operationId": "start_observed_sync_roi_calculator_observed_sync_post", + "parameters": [ + { + "in": "query", + "name": "days", + "required": false, + "schema": { + "anyOf": [ + { + "maximum": 366, + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Days" + } + } + ], + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ROISyncStatus" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Start Observed Sync", + "tags": [ + "roi_calculator" + ] + } + }, "/roi-calculator/report": { "get": { "operationId": "get_roi_calculator_report_roi_calculator_report_get", diff --git a/litellm/proxy/management_endpoints/roi_calculator_endpoints.py b/litellm/proxy/management_endpoints/roi_calculator_endpoints.py index bb6d60db723..4d2f53f2227 100644 --- a/litellm/proxy/management_endpoints/roi_calculator_endpoints.py +++ b/litellm/proxy/management_endpoints/roi_calculator_endpoints.py @@ -15,20 +15,29 @@ from apscheduler.schedulers.asyncio import ( # pyright: ignore[reportMissingTyp AsyncIOScheduler, ) from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query -from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError from starlette.types import Receive, Scope, Send from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params ) -from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.management_endpoints.roi_observed_endpoints import router as observed_router from litellm.proxy.roi_calculator.analytics import normalize_email, summarize from litellm.proxy.roi_calculator.branch_spend import BranchSpendDatabase, read_branch_spend from litellm.proxy.roi_calculator.estimator import CompletionCaller, EstimatorModel from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.settings import ( + active_connection, + get_roi_config_repository, + load_settings, + load_stored_settings, + read_admin, + save_settings, + write_admin, +) from litellm.proxy.roi_calculator.source import create_source from litellm.proxy.roi_calculator.sync import ( BranchSpendReader, @@ -62,29 +71,13 @@ from litellm.types.roi_calculator import ( ) router: Final = APIRouter() +router.include_router(observed_router) _SETTINGS_KEY: Final = "roi_calculator_settings" _REPORT_KEY: Final = "roi_calculator_report" _SYNC_MANAGER: Final = SyncManager() _ROI_TAGS: Final[list[str | Enum]] = ["roi calculator"] # mutable-ok: FastAPI requires list-valued route tags -class _StoredSettings(BaseModel): - model_config = ConfigDict(extra="ignore") - - source_provider: Literal["github", "gitlab"] = "github" - gitlab_api_url: str = "https://gitlab.com/api/v4" - gitlab_token: str = "" - github_api_url: str = "https://api.github.com" - github_token: str = "" - estimator_key: str = "" - repos: tuple[str, ...] = () - estimator_model: str = "" - estimator_prompt: str = DEFAULT_PROMPT - backfill_days: int = Field(default=7, ge=1, le=3650) - update_interval_minutes: float = Field(default=1440, ge=0, le=43200) - identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({})) - - class _RouterEstimatorParams(BaseModel): model_config = ConfigDict(extra="ignore", from_attributes=True) @@ -108,38 +101,6 @@ class _RouterEstimatorDeployment(BaseModel): model_info: _RouterEstimatorModelInfo | None = None -async def _read_admin( - user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -) -> UserAPIKeyAuth: - if user_api_key_dict.user_role not in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ): - raise HTTPException(status_code=403, detail="Only proxy admins can access the ROI Calculator.") - return user_api_key_dict - - -async def _write_admin( - user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -) -> UserAPIKeyAuth: - if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: - raise HTTPException(status_code=403, detail="Only proxy admins can change ROI Calculator settings.") - return user_api_key_dict - - -async def get_roi_config_repository( - _user: Annotated[UserAPIKeyAuth, Depends(_read_admin)], -) -> ConfigRepository: - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - raise HTTPException( - status_code=500, - detail=CommonProxyErrors.db_not_connected_error.value, - ) - return ConfigRepository(prisma_client, use_writer=True) - - def get_roi_sync_manager() -> SyncManager: return _SYNC_MANAGER @@ -222,66 +183,6 @@ def _router_estimator_choices() -> tuple[ROIEstimatorModel, ...]: return tuple(choice for choice in choices if choice.model_name in names) -async def _load_stored_settings(repository: ConfigRepository) -> _StoredSettings: - parameter: Final = await repository.get_param(_SETTINGS_KEY) - if parameter is None: - return _StoredSettings() - try: - return _StoredSettings.model_validate(parameter.param_value) - except ValidationError: - raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None - - -async def _load_settings(repository: ConfigRepository) -> ROISettings: - stored: Final = await _load_stored_settings(repository) - token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else "" - try: - return ROISettings( - source_provider=stored.source_provider, - gitlab_api_url=stored.gitlab_api_url, - gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "") - if stored.gitlab_token - else SecretStr(""), - github_api_url=stored.github_api_url, - github_token=SecretStr(token or ""), - estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "") - if stored.estimator_key - else SecretStr(""), - update_interval_minutes=stored.update_interval_minutes, - repos=stored.repos, - estimator_model=stored.estimator_model, - estimator_prompt=stored.estimator_prompt, - backfill_days=stored.backfill_days, - identity_map=stored.identity_map, - ) - except ValidationError: - raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None - - -async def _save_settings( - repository: ConfigRepository, - settings: ROISettings, - encrypted_token: str, - encrypted_estimator_key: str, - encrypted_gitlab_token: str = "", -) -> None: - stored: Final = _StoredSettings( - source_provider=settings.source_provider, - gitlab_api_url=settings.gitlab_api_url, - gitlab_token=encrypted_gitlab_token, - github_api_url=settings.github_api_url, - github_token=encrypted_token, - estimator_key=encrypted_estimator_key, - update_interval_minutes=settings.update_interval_minutes, - repos=settings.repos, - estimator_model=settings.estimator_model, - estimator_prompt=settings.estimator_prompt, - backfill_days=settings.backfill_days, - identity_map=settings.identity_map, - ) - await repository.set_param(_SETTINGS_KEY, stored.model_dump(mode="json")) - - async def _load_report(repository: ConfigRepository, settings: ROISettings) -> ROIReport | None: parameter: Final = await repository.get_param(_REPORT_KEY) if parameter is None or parameter.param_value is None: @@ -302,6 +203,7 @@ def _public_settings(settings: ROISettings) -> ROISettingsResponse: choices: Final = _router_estimator_choices() models: Final = tuple(choice.model_name for choice in choices) return ROISettingsResponse( + report_mode=settings.report_mode, source_provider=settings.source_provider, gitlab_api_url=settings.gitlab_api_url, has_gitlab_token=bool(settings.gitlab_token.get_secret_value()), @@ -392,14 +294,14 @@ async def _test_estimator_access(settings: ROISettings) -> None: raise HTTPException(status_code=409, detail="The estimator key could not connect to the gateway.") from None -def _gateway_user_reader(repository: ConfigRepository) -> GatewayUserReader: +def gateway_user_reader(repository: ConfigRepository) -> GatewayUserReader: async def get_emails() -> frozenset[str]: return await read_gateway_user_emails(spend_prisma_client(repository.prisma_client)) return get_emails -def _spend_reader(repository: ConfigRepository) -> SpendReader: +def spend_reader(repository: ConfigRepository) -> SpendReader: async def get_spend(start: date, end: date) -> tuple[ROISpendRecord, ...]: prisma_client: Final = spend_prisma_client(repository.prisma_client) return await read_spend(prisma_client, start, end) @@ -407,7 +309,7 @@ def _spend_reader(repository: ConfigRepository) -> SpendReader: return get_spend -def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader: +def branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> BranchSpendReader: async def get_spend(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]: return await read_branch_spend( cast( # cast-ok: PrismaWrapper delegates methods dynamically @@ -428,10 +330,10 @@ def _branch_spend_reader(repository: ConfigRepository, settings: ROISettings) -> tags=_ROI_TAGS, ) async def get_roi_calculator_settings( - _user: Annotated[UserAPIKeyAuth, Depends(_read_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], ) -> ROISettingsResponse: - return _public_settings(await _load_settings(repository)) + return _public_settings(await load_settings(repository)) @router.put( @@ -441,11 +343,11 @@ async def get_roi_calculator_settings( ) async def update_roi_calculator_settings( patch: ROISettingsUpdate, - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], ) -> ROISettingsResponse: - stored: Final = await _load_stored_settings(repository) - current: Final = await _load_settings(repository) + stored: Final = await load_stored_settings(repository) + current: Final = await load_settings(repository, stored) if "github_api_url" in patch.model_fields_set and patch.github_api_url is None: raise HTTPException(status_code=422, detail="GitHub API URL cannot be null.") if "gitlab_api_url" in patch.model_fields_set and patch.gitlab_api_url is None: @@ -491,6 +393,17 @@ async def update_roi_calculator_settings( ) try: settings: Final = ROISettings( + report_mode=patch.report_mode or current.report_mode, + connection_type="token" + if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set + else current.connection_type, + oauth_refresh_token=SecretStr("") + if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set + else current.oauth_refresh_token, + oauth_expires_at=None + if source_changed or token_was_supplied or "gitlab_token" in patch.model_fields_set + else current.oauth_expires_at, + ignored_logins=() if source_changed else current.ignored_logins, source_provider=provider, gitlab_api_url=gitlab_url, gitlab_token=SecretStr(gitlab_token), @@ -510,7 +423,15 @@ async def update_roi_calculator_settings( ) except ValidationError as exc: raise HTTPException(status_code=422, detail=exc.errors(include_context=False)) from None - await _save_settings(repository, settings, encrypted_token, encrypted_estimator_key, encrypted_gitlab) + await save_settings( + repository, + settings, + encrypted_token, + encrypted_estimator_key, + encrypted_gitlab, + revision=stored.revision, + replace_connection_id=active_connection(stored).id, + ) if source_changed: await repository.set_param(_REPORT_KEY, None) return _public_settings(settings) @@ -522,13 +443,13 @@ async def update_roi_calculator_settings( tags=_ROI_TAGS, ) async def get_roi_calculator_repositories( - _user: Annotated[UserAPIKeyAuth, Depends(_read_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)], query: Annotated[str, Query(max_length=200)] = "", page: Annotated[int, Query(ge=1, le=1000)] = 1, ) -> ROIRepositoriesResponse: - github: Final = create_source(await _load_settings(repository), transport) + github: Final = create_source(await load_settings(repository), transport) try: repos, has_more = await github.repositories(query, page) except SourceError as exc: @@ -550,12 +471,12 @@ async def get_roi_calculator_repositories( tags=_ROI_TAGS, ) async def get_roi_calculator_sync_status( - _user: Annotated[UserAPIKeyAuth, Depends(_read_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], manager: Annotated[SyncManager, Depends(get_roi_sync_manager)], ) -> ROISyncStatus: status: Final = await SyncStore(repository.prisma_client).status() or manager.status - settings: Final = await _load_settings(repository) + settings: Final = await load_settings(repository) report: Final = await _load_report(repository, settings) next_update: Final = _next_update(settings, status, report) return status.model_copy(update=MappingProxyType({"next_update": next_update.isoformat() if next_update else None})) @@ -568,25 +489,25 @@ async def get_roi_calculator_sync_status( tags=_ROI_TAGS, ) async def start_roi_calculator_sync( - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], manager: Annotated[SyncManager, Depends(get_roi_sync_manager)], transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)], ) -> ROISyncStatus: - settings: Final = await _load_settings(repository) + settings: Final = await load_settings(repository) public: Final = _public_settings(settings) if not public.ready: raise HTTPException(status_code=409, detail="Connect a source, select repositories, and choose a router model.") if not await manager.start( settings, repository, - _spend_reader(repository), + spend_reader(repository), _completion_caller(settings), transport, _router_estimator_models(settings.estimator_model), SyncStore(repository.prisma_client), - branch_spend_reader=_branch_spend_reader(repository, settings), - gateway_user_reader=_gateway_user_reader(repository), + branch_spend_reader=branch_spend_reader(repository, settings), + gateway_user_reader=gateway_user_reader(repository), ): raise HTTPException(status_code=409, detail="A sync is already running.") return manager.status @@ -598,7 +519,7 @@ async def start_roi_calculator_sync( tags=_ROI_TAGS, ) async def cancel_roi_calculator_sync( - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], manager: Annotated[SyncManager, Depends(get_roi_sync_manager)], ) -> ROISyncStatus: @@ -614,7 +535,7 @@ async def cancel_roi_calculator_sync( tags=_ROI_TAGS, ) async def get_roi_calculator_report( - _user: Annotated[UserAPIKeyAuth, Depends(_read_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], mode: Literal["live", "demo"] = "live", ) -> ROIReportResponse: @@ -623,7 +544,7 @@ async def get_roi_calculator_report( sample: Final = summarize(sample_report(datetime.now(timezone.utc)), MappingProxyType({})) return ROIReportResponse(report=ROISummaryResponse.model_validate(sample)) - settings: Final = await _load_settings(repository) + settings: Final = await load_settings(repository) report: Final = await _load_report(repository, settings) if report is None: return ROIReportResponse(report=None) @@ -638,12 +559,12 @@ async def get_roi_calculator_report( ) async def update_roi_calculator_identity_map( update: ROIIdentityMapUpdate, - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], ) -> ROIIdentityMapResponse: login: Final = update.github_login.strip().casefold() - current: Final = await _load_settings(repository) - current_stored: Final = await _load_stored_settings(repository) + current_stored: Final = await load_stored_settings(repository) + current: Final = await load_settings(repository, current_stored) try: normalize_source_login(login, current.source_provider) except ValueError as exc: @@ -657,6 +578,11 @@ async def update_roi_calculator_identity_map( else MappingProxyType({**current.identity_map, login: new_email}) ) settings: Final = ROISettings( + report_mode=current.report_mode, + connection_type=current.connection_type, + oauth_refresh_token=current.oauth_refresh_token, + oauth_expires_at=current.oauth_expires_at, + ignored_logins=current.ignored_logins, source_provider=current.source_provider, gitlab_api_url=current.gitlab_api_url, gitlab_token=current.gitlab_token, @@ -670,8 +596,13 @@ async def update_roi_calculator_identity_map( backfill_days=current.backfill_days, identity_map=identity_map, ) - await _save_settings( - repository, settings, current_stored.github_token, current_stored.estimator_key, current_stored.gitlab_token + await save_settings( + repository, + settings, + current_stored.github_token, + current_stored.estimator_key, + current_stored.gitlab_token, + revision=current_stored.revision, ) report: Final = await _load_report(repository, settings) summary: Final = summarize(report, settings.identity_map) if report is not None else None @@ -683,7 +614,8 @@ async def update_roi_calculator_identity_map( def _next_update(settings: ROISettings, status: ROISyncStatus, report: ROIReport | None) -> datetime | None: if ( - not report + settings.report_mode != "legacy" + or not report or not settings.repos or not settings.estimator_model or not settings.update_interval_minutes @@ -710,12 +642,16 @@ def register_scheduled_sync(scheduler: AsyncIOScheduler) -> None: async def run_scheduled_sync() -> None: + from litellm.proxy.management_endpoints.roi_observed_endpoints import run_observed_schedule from litellm.proxy.proxy_server import prisma_client if prisma_client is None: return repository: Final = ConfigRepository(prisma_client, use_writer=True) - settings: Final = await _load_settings(repository) + settings: Final = await load_settings(repository) + if settings.report_mode == "observed": + await run_observed_schedule() + return if not settings.update_interval_minutes or not _public_settings(settings).ready: return store: Final = SyncStore(prisma_client) @@ -727,23 +663,23 @@ async def run_scheduled_sync() -> None: await _SYNC_MANAGER.start( settings, repository, - _spend_reader(repository), + spend_reader(repository), _completion_caller(settings), estimator_models=_router_estimator_models(settings.estimator_model), coordinator=store, scheduled_interval=settings.update_interval_minutes, - branch_spend_reader=_branch_spend_reader(repository, settings), - gateway_user_reader=_gateway_user_reader(repository), + branch_spend_reader=branch_spend_reader(repository, settings), + gateway_user_reader=gateway_user_reader(repository), ) @router.post("/roi-calculator/connections/test", tags=_ROI_TAGS) async def test_roi_calculator_connections( - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_github_transport)], ) -> ROISettingsResponse: - settings: Final = await _load_settings(repository) + settings: Final = await load_settings(repository) public: Final = _public_settings(settings) if not public.ready: raise HTTPException(status_code=409, detail="Choose repositories and an available estimator model first.") @@ -760,7 +696,7 @@ async def test_roi_calculator_connections( @router.post("/roi-calculator/setup/reset", tags=_ROI_TAGS) async def reset_roi_calculator_setup( - _user: Annotated[UserAPIKeyAuth, Depends(_write_admin)], + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], ) -> ROISettingsResponse: from uuid import uuid4 @@ -781,10 +717,17 @@ async def reset_roi_calculator_setup( if not await store.acquire(owner, status): raise HTTPException(status_code=409, detail="Cancel the running analysis before restarting setup.") try: - current: Final = await _load_settings(repository) - stored: Final = await _load_stored_settings(repository) + stored: Final = await load_stored_settings(repository) + current: Final = await load_settings(repository, stored) settings: Final = current.model_copy(update=MappingProxyType({"repos": ()})) - await _save_settings(repository, settings, stored.github_token, stored.estimator_key, stored.gitlab_token) + await save_settings( + repository, + settings, + stored.github_token, + stored.estimator_key, + stored.gitlab_token, + revision=stored.revision, + ) await store.clear_report() return _public_settings(settings) finally: diff --git a/litellm/proxy/management_endpoints/roi_observed_endpoints.py b/litellm/proxy/management_endpoints/roi_observed_endpoints.py new file mode 100644 index 00000000000..424d7af4076 --- /dev/null +++ b/litellm/proxy/management_endpoints/roi_observed_endpoints.py @@ -0,0 +1,587 @@ +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from typing import Annotated, Final + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi.responses import JSONResponse, RedirectResponse +from pydantic import SecretStr, TypeAdapter, ValidationError + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.roi_calculator.analytics import normalize_email +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.oauth import ( + OAuthConfig, + Provider, + begin_authorization, + connected_settings, + consume_state, + exchange_code, + oauth_config, + save_grant, +) +from litellm.proxy.roi_calculator.observed_sync import ObservedSyncManager, Progress +from litellm.proxy.roi_calculator.observed_workspace import ( + collect_workspace, + scoped_data, + source_details, + summarize_workspace, +) +from litellm.proxy.roi_calculator.settings import ( + StoredConnection, + active_connection, + connection_id, + enable_observed_reporting, + get_roi_config_repository, + load_settings, + load_stored_settings, + read_admin, + save_connection_identities, + save_settings, + select_connection, + stored_connections, + write_admin, +) +from litellm.proxy.roi_calculator.source import create_source +from litellm.proxy.roi_calculator.sync import read_gateway_user_emails, spend_prisma_client +from litellm.proxy.roi_calculator.sync_store import SyncStore +from litellm.repositories.config_repository import ConfigRepository +from litellm.types.roi_calculator import ( + ROIRepositoriesResponse, + ROIRepository, + ROISettings, + ROISyncStatus, + normalize_source_login, +) +from litellm.types.roi_observed import ( + ObservedAccount, + ObservedApp, + ObservedApps, + ObservedAuthorization, + ObservedConnectionIdentities, + ObservedData, + ObservedIdentities, + ObservedIdentityUpdate, + ObservedReportResponse, + ObservedSettings, + ObservedSettingsUpdate, +) + +router: Final = APIRouter(prefix="/roi-calculator/observed", tags=["roi calculator"]) +_MANAGER: Final = ObservedSyncManager() +_REPORT_KEY: Final = "roi_observed_report" + + +def get_observed_manager() -> ObservedSyncManager: + return _MANAGER + + +def get_observed_transport() -> httpx.AsyncBaseTransport | None: + return None + + +def public_settings(settings: ROISettings) -> ObservedSettings: + return ObservedSettings( + id=connection_id(settings.source_provider, settings.source_api_url), + source_provider=settings.source_provider, + api_url=settings.source_api_url, + repos=settings.repos, + has_token=bool( + ( + settings.gitlab_token if settings.source_provider == "gitlab" else settings.github_token + ).get_secret_value() + ), + update_interval_minutes=settings.update_interval_minutes, + ready=bool(settings.repos), + connection_type=settings.connection_type, + ) + + +async def workspace_settings(repository: ConfigRepository) -> ObservedSettings: + stored: Final = await load_stored_settings(repository) + entries: Final = tuple( + [ + public_settings(await load_settings(repository, select_connection(stored, entry))) + for entry in stored_connections(stored) + ] + ) + current: Final = public_settings(await load_settings(repository, stored)) + return current.model_copy(update={"connections": entries, "ready": any(entry.ready for entry in entries)}) + + +async def _data(repository: ConfigRepository) -> ObservedData | None: + saved: Final = await repository.get_param(_REPORT_KEY) + if saved is None or saved.param_value is None: + return None + try: + data: Final = ObservedData.model_validate(saved.param_value) + except ValidationError: + raise HTTPException(500, "The saved report is invalid. Sync again to rebuild it.") from None + if data.connections: + return data + settings: Final = ROISettings.model_validate( + { + "source_provider": data.source_provider, + "repos": data.repos, + ("gitlab_api_url" if data.source_provider == "gitlab" else "github_api_url"): data.source_api_url, + } + ) + return scoped_data(data, source_details(settings)) + + +@router.get("/settings", response_model=ObservedSettings) +async def get_observed_settings( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], +) -> ObservedSettings: + return await workspace_settings(repository) + + +@router.put("/settings", response_model=ObservedSettings) +async def save_observed_settings( + patch: ObservedSettingsUpdate, + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)], +) -> ObservedSettings: + if (status := await SyncStore(repository.prisma_client, "roi_observed").status()) and status.running: + raise HTTPException(409, "Cancel the running sync before changing the connection.") + original: Final = await load_stored_settings(repository) + target_id: Final = connection_id(patch.source_provider, patch.api_url) + selected_id: Final = patch.connection_id or target_id + selected: Final = next((entry for entry in stored_connections(original) if entry.id == selected_id), None) + if patch.connection_id and selected is None: + raise HTTPException(404, "This connection no longer exists. Reload Connections.") + if ( + patch.connection_id + and selected_id != target_id + and any(entry.id == target_id for entry in stored_connections(original)) + ): + raise HTTPException(409, "This provider and host are already connected. Edit that connection instead.") + if patch.token is None and selected: + await connected_settings(repository, transport, selected_id) + refreshed: Final = await load_stored_settings(repository) + saved_connection: Final = next((entry for entry in stored_connections(refreshed) if entry.id == selected_id), None) + stored: Final = select_connection(refreshed, saved_connection) if saved_connection else refreshed + current: Final = await load_settings(repository, stored) + changed: Final = target_id != connection_id(current.source_provider, current.source_api_url) + existing_token: Final = current.gitlab_token if patch.source_provider == "gitlab" else current.github_token + token: Final = patch.token if patch.token is not None else "" if changed else existing_token.get_secret_value() + updates: Final[Mapping[str, object]] = { + "report_mode": "observed", + "source_provider": patch.source_provider, + "repos": patch.repos, + "update_interval_minutes": patch.update_interval_minutes + if patch.update_interval_minutes is not None + else current.update_interval_minutes, + "identity_map": {} if changed else current.identity_map, + "ignored_logins": () if changed else current.ignored_logins, + "connection_type": "token" if patch.token is not None or changed else current.connection_type, + "oauth_refresh_token": SecretStr("") if patch.token is not None or changed else current.oauth_refresh_token, + "oauth_expires_at": None if patch.token is not None or changed else current.oauth_expires_at, + ("gitlab_api_url" if patch.source_provider == "gitlab" else "github_api_url"): patch.api_url, + ("gitlab_token" if patch.source_provider == "gitlab" else "github_token"): SecretStr(token), + } + try: + settings: Final = ROISettings.model_validate({**current.model_dump(), **updates}) + except ValidationError as exc: + raise HTTPException(422, exc.errors(include_context=False, include_input=False)) from None + source: Final = create_source(settings, transport) + try: + if settings.repos: + await source.test_repositories(settings.repos) + elif token: + await source.repositories(page=1) + except SourceError as exc: + raise HTTPException(502, str(exc)) from None + finally: + await source.close() + encrypted: Final = TypeAdapter(str).validate_python(encrypt_value_helper(token)) if token else "" + await save_settings( + repository, + settings, + encrypted if settings.source_provider == "github" else stored.github_token, + stored.estimator_key, + encrypted if settings.source_provider == "gitlab" else stored.gitlab_token, + revision=stored.revision, + replace_connection_id=patch.connection_id, + ) + return await workspace_settings(repository) + + +@router.get("/report", response_model=ObservedReportResponse) +async def get_observed_report( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], +) -> ObservedReportResponse: + stored: Final = await load_stored_settings(repository) + data: Final = await _data(repository) + return ObservedReportResponse(report=summarize_workspace(data, stored_connections(stored)) if data else None) + + +@router.get("/identities", response_model=ObservedIdentities) +async def get_observed_identities( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], +) -> ObservedIdentities: + stored: Final = await load_stored_settings(repository) + data: Final = await _data(repository) + report: Final = summarize_workspace(data, stored_connections(stored)) if data else None + return ObservedIdentities( + gateway_emails=tuple(sorted(await read_gateway_user_emails(spend_prisma_client(repository.prisma_client)))), + identity_map=stored.identity_map, + unmatched_logins=report.unmatched_logins if report else (), + connections=tuple( + ObservedConnectionIdentities( + id=entry.id, + source_provider=entry.source_provider, + api_url=entry.api_url, + repos=entry.repos, + identity_map=entry.identity_map, + unmatched_logins=tuple( + login.split(":", 1)[-1] for login in report.unmatched_logins if login.startswith(entry.id + ":") + ) + if report + else (), + ) + for entry in stored_connections(stored) + ), + ) + + +@router.put("/identities", response_model=ObservedReportResponse) +async def save_observed_identities( + patch: ObservedIdentityUpdate, + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], +) -> ObservedReportResponse: + email: Final = normalize_email(patch.email) + emails: Final = await read_gateway_user_emails(spend_prisma_client(repository.prisma_client)) + if email not in emails: + raise HTTPException(422, "Choose an existing internal user email.") + stored: Final = await load_stored_settings(repository) + connections: Final = stored_connections(stored) + if not connections: + raise HTTPException(409, "Connect a repository before linking accounts.") + accounts: Final = ( + patch.accounts + if patch.accounts is not None + else tuple(ObservedAccount(connection_id=active_connection(stored).id, login=login) for login in patch.logins) + ) + if any(account.connection_id not in {entry.id for entry in connections} for account in accounts): + raise HTTPException(422, "Choose an existing connection.") + data: Final = await _data(repository) + report: Final = summarize_workspace(data, connections) if data else None + existing: Final = next((person.accounts for person in report.people if person.email == email), ()) if report else () + + def update(entry: StoredConnection) -> StoredConnection: + if patch.accounts is None and entry.id != active_connection(stored).id: + return entry + try: + logins: Final = tuple( + normalize_source_login(account.login, entry.source_provider) + for account in accounts + if account.connection_id == entry.id + ) + except ValueError as exc: + raise HTTPException(422, str(exc)) from None + if any(login in entry.identity_map and entry.identity_map[login] != email for login in logins): + raise HTTPException(409, "An account is already linked to another email. Unlink it first.") + old: Final = frozenset( + ( + *(login for login, address in entry.identity_map.items() if address == email), + *(account.login for account in existing if account.connection_id == entry.id), + ) + ) + return entry.model_copy( + update={ + "identity_map": { + **{login: address for login, address in entry.identity_map.items() if address != email}, + **dict.fromkeys(logins, email), + }, + "ignored_logins": tuple(sorted((frozenset(entry.ignored_logins) | old) - frozenset(logins))), + } + ) + + updated: Final = tuple(update(entry) for entry in connections) + await save_connection_identities(repository, stored, updated) + return ObservedReportResponse(report=summarize_workspace(data, updated) if data else None) + + +@router.get("/sync", response_model=ROISyncStatus) +async def get_observed_sync( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)], +) -> ROISyncStatus: + return await SyncStore(repository.prisma_client, "roi_observed").status() or manager.status + + +async def _report_days(repository: ConfigRepository, days: int | None) -> int: + if days is not None: + return days + saved: Final = await repository.get_param(_REPORT_KEY) + if saved is None: + return 28 + try: + data: Final = ObservedData.model_validate(saved.param_value) + except ValidationError: + return 28 + return (data.current.window.end - data.current.window.start).days + 1 + + +async def _start_sync( + repository: ConfigRepository, + manager: ObservedSyncManager, + transport: httpx.AsyncBaseTransport | None, + scheduled_interval: float = 0, + days: int | None = None, +) -> bool: + from litellm.proxy.management_endpoints.roi_calculator_endpoints import branch_spend_reader + + stored: Final = await load_stored_settings(repository) + reporting_days: Final = await _report_days(repository, days) + entries: Final = tuple(entry for entry in stored_connections(stored) if entry.repos) + if not entries: + raise HTTPException(409, "Select at least one repository.") + settings: Final = tuple([await connected_settings(repository, transport, entry.id) for entry in entries]) + await enable_observed_reporting(repository) + + async def build(progress: Progress) -> ObservedData: + from litellm.proxy.management_endpoints.roi_calculator_endpoints import gateway_user_reader, spend_reader + + data: Final = await collect_workspace( + tuple((entry, branch_spend_reader(repository, entry)) for entry in settings), + spend_reader(repository), + gateway_user_reader(repository), + datetime.now(timezone.utc), + progress, + transport, + days=reporting_days, + ) + current: Final = tuple( + entry for entry in stored_connections(await load_stored_settings(repository)) if entry.repos + ) + if tuple((entry.id, entry.repos) for entry in current) != tuple((entry.id, entry.repos) for entry in entries): + raise SourceError("The connection changed during sync. Sync again with the current repositories.") + return data + + return await manager.start(build, SyncStore(repository.prisma_client, "roi_observed"), scheduled_interval) + + +@router.post("/sync", response_model=ROISyncStatus, status_code=202) +async def start_observed_sync( + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)], + transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)], + days: Annotated[int | None, Query(ge=1, le=366)] = None, +) -> ROISyncStatus: + if not await _start_sync(repository, manager, transport, days=days): + raise HTTPException(409, "A sync is already running.") + return manager.status + + +@router.delete("/sync", response_model=ROISyncStatus) +async def cancel_observed_sync( + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + manager: Annotated[ObservedSyncManager, Depends(get_observed_manager)], +) -> ROISyncStatus: + store: Final = SyncStore(repository.prisma_client, "roi_observed") + await store.cancel() + await manager.cancel() + return await store.status() or manager.status + + +async def run_observed_schedule() -> None: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return + repository: Final = ConfigRepository(prisma_client, use_writer=True) + settings: Final = await load_settings(repository) + if ( + not any(entry.repos for entry in stored_connections(await load_stored_settings(repository))) + or not settings.update_interval_minutes + ): + return + status: Final = await SyncStore(prisma_client, "roi_observed").status() + if status and status.running: + return + anchor: Final = status.finished_at if status else None + if anchor and datetime.fromisoformat(anchor) + timedelta(minutes=settings.update_interval_minutes) > datetime.now( + timezone.utc + ): + return + await _start_sync(repository, _MANAGER, None, settings.update_interval_minutes) + + +@router.get("/repositories", response_model=ROIRepositoriesResponse) +async def observed_repositories( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)], + connection: Annotated[str | None, Query(max_length=100)] = None, + query: Annotated[str, Query(max_length=200)] = "", + page: Annotated[int, Query(ge=1, le=1000)] = 1, +) -> ROIRepositoriesResponse: + source: Final = create_source(await connected_settings(repository, transport, connection), transport) + try: + repositories, more = await source.repositories(query, page) + except SourceError as exc: + raise HTTPException(502, str(exc)) from None + finally: + await source.close() + return ROIRepositoriesResponse( + repositories=tuple( + ROIRepository(name=name, visibility=visibility, archived=archived) + for name, visibility, archived in repositories + ), + page=page, + has_more=more, + ) + + +@router.get("/apps", response_model=ObservedApps) +async def observed_apps(_user: Annotated[UserAPIKeyAuth, Depends(read_admin)]) -> ObservedApps: + def details(provider: Provider) -> ObservedApp: + config: Final = oauth_config(provider) + return ObservedApp( + configured=config is not None, + can_install=bool(config and config.installation_url), + api_url=config.api_url if config else None, + callback_url=config.redirect_uri if config else None, + ) + + return ObservedApps(github=details("github"), gitlab=details("gitlab")) + + +@router.post("/oauth/{provider}/start", response_model=ObservedAuthorization) +async def start_observed_authorization( + provider: Provider, + _user: Annotated[UserAPIKeyAuth, Depends(write_admin)], + repository: Annotated[ConfigRepository, Depends(get_roi_config_repository)], + install: bool = False, +) -> JSONResponse: + config: Final = oauth_config(provider) + if config is None: + raise HTTPException(409, "Configure the provider app client ID, client secret, and PROXY_BASE_URL first.") + if install and not config.installation_url: + raise HTTPException(409, "Configure the GitHub app slug to manage repository access.") + url, nonce = await begin_authorization(repository, config, install=install) + response: Final = JSONResponse(ObservedAuthorization(url=url).model_dump(mode="json")) + response.set_cookie( + "litellm_roi_oauth", + nonce, + httponly=True, + secure=config.proxy_url.startswith("https://"), + samesite="lax", + max_age=600, + path=config.cookie_path, + ) + response.headers["Cache-Control"] = "no-store" + return response + + +def get_oauth_repository() -> ConfigRepository: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(503, "The database is unavailable. Try connecting again later.") + return ConfigRepository(prisma_client, use_writer=True) + + +def _authorization_redirect(config: OAuthConfig, query: str) -> RedirectResponse: + response: Final = RedirectResponse(config.proxy_url + "/ui/roi-calculator/?" + query, status_code=303) + response.delete_cookie("litellm_roi_oauth", path=config.cookie_path) + response.headers["Cache-Control"] = "no-store" + response.headers["Referrer-Policy"] = "no-referrer" + return response + + +@router.get("/oauth/{provider}/callback", include_in_schema=False) +async def observed_authorization_callback( + provider: Provider, + request: Request, + repository: Annotated[ConfigRepository, Depends(get_oauth_repository)], + transport: Annotated[httpx.AsyncBaseTransport | None, Depends(get_observed_transport)], + state: str = "", + code: str = "", + error: str = "", +) -> RedirectResponse: + config: Final = oauth_config(provider) + if config is None: + raise HTTPException(409, "The provider app is not configured.") + try: + return await _complete_authorization(config, request, repository, transport, state, code, error) + except HTTPException: + return _authorization_redirect(config, "connection_failed=1") + + +async def _complete_authorization( + config: OAuthConfig, + request: Request, + repository: ConfigRepository, + transport: httpx.AsyncBaseTransport | None, + state: str, + code: str, + error: str, +) -> RedirectResponse: + verified: Final = await consume_state(repository, state, request.cookies.get("litellm_roi_oauth", ""), config) + if error or not code: + return _authorization_redirect(config, "connection_cancelled=1") + if (status := await SyncStore(repository.prisma_client, "roi_observed").status()) and status.running: + raise HTTPException(409, "Cancel the running sync before changing the connection.") + grant: Final = await exchange_code(config, verified, code, transport) + validation_settings: Final = ROISettings.model_validate( + { + "source_provider": config.provider, + "connection_type": "app", + ("github_api_url" if config.provider == "github" else "gitlab_api_url"): config.api_url, + ("github_token" if config.provider == "github" else "gitlab_token"): grant.access_token, + } + ) + source: Final = create_source(validation_settings, transport) + try: + await source.repositories(page=1) + except SourceError as exc: + raise HTTPException(502, str(exc)) from None + finally: + await source.close() + await save_grant(repository, config, grant, revision=verified.settings_revision) + return _authorization_redirect(config, "connected=" + config.provider) + + +@router.get("/oauth/github/installed", include_in_schema=False) +async def observed_installation_callback( + request: Request, + repository: Annotated[ConfigRepository, Depends(get_oauth_repository)], + state: str = "", +) -> RedirectResponse: + config: Final = oauth_config("github") + if config is None or config.installation_url is None: + raise HTTPException(409, "The GitHub app is not configured.") + try: + verified: Final = await consume_state( + repository, state, request.cookies.get("litellm_roi_oauth", ""), config, flow="install" + ) + if (await load_stored_settings(repository)).revision != verified.settings_revision: + raise HTTPException(409, "The connection changed during installation. Start again from Connections.") + url, nonce = await begin_authorization(repository, config) + except HTTPException: + return _authorization_redirect(config, "connection_failed=1") + response: Final = RedirectResponse(url, status_code=303) + response.set_cookie( + "litellm_roi_oauth", + nonce, + httponly=True, + secure=config.proxy_url.startswith("https://"), + samesite="lax", + max_age=600, + path=config.cookie_path, + ) + response.headers["Cache-Control"] = "no-store" + response.headers["Referrer-Policy"] = "no-referrer" + return response diff --git a/litellm/proxy/roi_calculator/README.md b/litellm/proxy/roi_calculator/README.md new file mode 100644 index 00000000000..4db62fdad2b --- /dev/null +++ b/litellm/proxy/roi_calculator/README.md @@ -0,0 +1,52 @@ +# ROI Calculator + +The dashboard compares merged pull or merge requests, elapsed time from opening to merge, new bug and regression issues, and recorded gateway spend over 7, 28, or 90 complete UTC days. Compare against the immediately preceding period of the same length or the same-length period last year + +The calculator combines repository activity with spend recorded by the gateway. Spend per merged change is a person's recorded gateway spend during the period divided by their matched merged changes. To track a branch's AI cost, send repository and branch tags with each request + +## Connect repositories + +Use **Preview sample report** beside the title to explore the dashboard before connecting repositories. Sample periods, engineer details, quality signals, and branch spend work without changing your connections or live report. **Exit demo** returns to your report or setup + +Open `/ui/roi-calculator/`, choose GitHub or GitLab, then connect with an app or access token. Select several repositories and start the sync. Use **Add connection** to keep both providers connected. Each provider and API host retains its credentials, repositories, and identity mappings, and the report combines their activity while counting each person’s gateway spend once. Public repositories also accept an empty token, subject to the provider's anonymous API limits + +For GitHub tokens, grant read access to metadata, pull requests and issues. GitLab tokens require `read_api`. Self-hosted instances use their API URL, for example `https://git.example.com/api/v4` + +## Configure app authorization + +Register a GitHub App with read-only repository permissions for metadata, pull requests and issues. Enable expiring user access tokens and leave authorization during installation disabled, since the gateway starts authorization after installation. Generate a private key in the app settings to allow installation and store it securely. The gateway uses a generated client secret for authorization and does not need the private key + +Register a confidential GitLab OAuth application with `read_api` and `read_user` scopes + +Set `PROXY_BASE_URL` to the gateway's public URL. The callback URLs are `/roi-calculator/observed/oauth/github/callback` and `/roi-calculator/observed/oauth/gitlab/callback` + +Set the GitHub App setup URL to `/roi-calculator/observed/oauth/github/installed`, enable **Redirect on update**, and set `LITELLM_ROI_GITHUB_APP_SLUG` to its URL slug. The first connection then starts with repository installation and continues to user authorization + +Set `LITELLM_ROI_GITHUB_CLIENT_ID` and `LITELLM_ROI_GITHUB_CLIENT_SECRET` for GitHub, or `LITELLM_ROI_GITLAB_CLIENT_ID` and `LITELLM_ROI_GITLAB_CLIENT_SECRET` for GitLab. For a self-hosted provider, set `LITELLM_ROI_GITHUB_URL` or `LITELLM_ROI_GITLAB_URL` to its base URL without the API suffix + +The gateway encrypts access and refresh tokens using its configured encryption key. Authorization uses PKCE and an expiring, single-use state tied to an HTTP-only browser cookie. Refreshes are coordinated across gateway workers + +## Link people + +Use **Link accounts** to associate several current or historical usernames with one internal email. Each connection has a separate username field, so a GitHub username never matches a GitLab user implicitly. Saving immediately recalculates the report without fetching repositories again. Public profile emails match automatically when they resolve unambiguously to an internal user + +Agent-authored changes count for a person only when the supported agent metadata explicitly names a requester. Repository issue counts and revert titles are quality signals, not an individual defect score + +Bug and regression counts combine repositories with issue tracking enabled. They remain unavailable when none of the selected repositories has issue tracking enabled + +## Sync behavior + +The default refresh interval is daily and applies to every connection in the workspace. Adding or editing a connection preserves it unless `update_interval_minutes` is supplied. The observed settings API accepts `update_interval_minutes: 0` for manual updates. A cancelled or failed sync preserves the last complete report + +Existing settings retain `report_mode: legacy` and their scheduled reports until an administrator saves a connection, authorizes an app, or starts an observed sync. Reading the new dashboard alone does not change the mode. The legacy settings API can explicitly select `report_mode: legacy` again + +GitHub collection splits large searches into smaller date ranges to avoid its search-result limit. Both providers validate pagination and reject incomplete responses instead of publishing partial counts + + +## Branch request tags + +Send `repo:github.com/owner/repo` or `repo:gitlab.com/group/project` together with `branch:feature/name` in `metadata.tags`, top-level `tags`, or the comma-separated `x-litellm-tags` header. The tags must identify the source repository and branch, including forks + +The report sums recorded requests inside its UTC dates. A branch cost is assigned to a merged change only when that source branch matches one change in the period. Reused branches stay visible in Branch spend without duplicating costs across changes. No retained tagged requests means unknown cost; a recorded zero remains zero + +An empty repository produces a successful report with zero merged changes and no merge duration or spend-per-change ratio diff --git a/litellm/proxy/roi_calculator/branch_spend.py b/litellm/proxy/roi_calculator/branch_spend.py index f441683e843..d2193a6cc01 100644 --- a/litellm/proxy/roi_calculator/branch_spend.py +++ b/litellm/proxy/roi_calculator/branch_spend.py @@ -61,12 +61,23 @@ async def read_branch_spend( def attribute_branches( pulls: tuple[ROIPullRecord, ...], spend: tuple[ROIBranchSpend, ...] | None ) -> Mapping[tuple[str, int], ROIBranchAttribution]: - counts: Final = Counter((pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls) + return attribute_branch_keys( + tuple( + (pull["repo"], pull["number"], pull.get("source_repo", ""), pull.get("source_branch", "")) for pull in pulls + ), + spend, + ) + + +def attribute_branch_keys( + pulls: tuple[tuple[str, int, str, str], ...], spend: tuple[ROIBranchSpend, ...] | None +) -> Mapping[tuple[str, int], ROIBranchAttribution]: + counts: Final = Counter((pull[2], pull[3]) for pull in pulls) costs: Final = {(row.repo, row.branch): row for row in spend or ()} - def attribute(pull: ROIPullRecord) -> ROIBranchAttribution: - repo: Final = pull.get("source_repo", "") - branch: Final = pull.get("source_branch", "") + def attribute(pull: tuple[str, int, str, str]) -> ROIBranchAttribution: + repo: Final = pull[2] + branch: Final = pull[3] cost: Final = costs.get((repo, branch)) if spend is None: return ROIBranchAttribution(repo=repo, branch=branch, status="unavailable") @@ -78,4 +89,4 @@ def attribute_branches( repo=repo, branch=branch, spend=cost.spend, requests=cost.requests, status="matched" ) - return {(pull["repo"], pull["number"]): attribute(pull) for pull in pulls} + return {(pull[0], pull[1]): attribute(pull) for pull in pulls} diff --git a/litellm/proxy/roi_calculator/github.py b/litellm/proxy/roi_calculator/github.py index 9f8aa8c26ae..993b6b9c9fa 100644 --- a/litellm/proxy/roi_calculator/github.py +++ b/litellm/proxy/roi_calculator/github.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import AsyncIterator, Mapping -from datetime import date +from datetime import date, datetime from types import MappingProxyType from typing import Final, TypeVar from urllib.parse import quote @@ -15,6 +15,7 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.roi_calculator.analytics import normalize_email from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings +from litellm.types.roi_observed import ObservedIssue _T: Final = TypeVar("_T") @@ -29,6 +30,8 @@ class _GitHubModel(BaseModel): class _GitHubUser(_GitHubModel): login: str | None = None + type: str = "User" + email: str | None = None class _GitHubHeadRepository(_GitHubModel): @@ -50,6 +53,7 @@ class GitHubPullListItem(_GitHubModel): body: str | None = None head: _GitHubHead | None = None user: _GitHubUser | None = None + created_at: datetime | None = None class _RepositoryItem(_GitHubModel): @@ -215,15 +219,18 @@ _GRAPHQL_QUERY: Final = """query($owner:String!, $name:String!, $number:Int!, $c }""" -async def _request( +async def request_github( client: httpx.AsyncClient, method: str, path: str, params: Mapping[str, str | int] | None = None, json_body: object | None = None, headers: Mapping[str, str] | None = None, + *, + read_only: bool = False, ) -> httpx.Response: async def send(attempt: int) -> httpx.Response: + retryable: Final = (method == "GET" or read_only) and attempt < 2 try: response: Final = await client.request( method, @@ -233,8 +240,11 @@ async def _request( headers=headers, ) except httpx.RequestError: + if retryable: + await asyncio.sleep(0.5 * (attempt + 1)) + return await send(attempt + 1) raise SourceError("Could not reach GitHub. Check the API URL and network connection.") from None - if response.status_code in (429, 502, 503, 504) and method == "GET" and attempt < 2: + if response.status_code in (429, 502, 503, 504) and retryable: await asyncio.sleep(0.5 * (attempt + 1)) return await send(attempt + 1) if response.status_code >= 400: @@ -267,7 +277,7 @@ async def _fetch_page( headers: Mapping[str, str] | None = None, error_message: str = "GitHub returned an unexpected pagination response.", ) -> tuple[tuple[_T, ...], bool]: - response: Final = await _request( + response: Final = await request_github( client, "GET", path, @@ -312,6 +322,21 @@ class _GitHubUserProfile(_GitHubModel): email: str | None = None +class _IssueLabel(_GitHubModel): + name: str + + +class _Issue(_GitHubModel): + number: int + created_at: datetime + labels: tuple[_IssueLabel, ...] = () + pull_request: object | None = None + + +class GitHubIssueSettings(BaseModel): + has_issues: bool + + class GitHub: def __init__( self, @@ -408,8 +433,8 @@ class GitHub: async def test_repositories(self, repos: tuple[str, ...]) -> None: for repo in repos: - await _request(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers) - await _request( + await request_github(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers) + await request_github( self.client, "GET", self._url(f"repos/{repo}/pulls"), @@ -438,8 +463,40 @@ class GitHub: return await _collect(matching_pulls()) + async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None: + response: Final = await request_github(self.client, "GET", self._url(f"repos/{repo}"), headers=self._headers) + try: + settings: Final = GitHubIssueSettings.model_validate(response.json()) + except ValueError: + raise SourceError("GitHub returned invalid repository settings.") from None + if not settings.has_issues: + return None + + async def matching_issues() -> AsyncIterator[ObservedIssue]: + async for page in _pages( + self.client, + self._url(f"repos/{repo}/issues"), + TypeAdapter(tuple[_Issue, ...]), + MappingProxyType( + {"state": "all", "sort": "created", "direction": "desc", "since": f"{start}T00:00:00Z"} + ), + headers=self._headers, + ): + for issue in page: + if issue.pull_request is None and start <= issue.created_at.date() <= end: + yield ObservedIssue( + repo=repo, + number=issue.number, + created_at=issue.created_at, + labels=tuple(label.name for label in issue.labels), + ) + if page and page[-1].created_at.date() < start: + return + + return await _collect(matching_issues()) + async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence: - detail_response: Final = await _request( + detail_response: Final = await request_github( self.client, "GET", self._url(f"repos/{repo}/pulls/{pull.number}"), @@ -574,7 +631,7 @@ class GitHub: ) -> tuple[tuple[ROIPullCommit, ...], tuple[tuple[str, str], ...], int]: if remaining_pages == 0: raise SourceError("GitHub commit pagination limit was reached.") - response: Final = await _request( + response: Final = await request_github( self.client, "POST", endpoint, @@ -583,6 +640,7 @@ class GitHub: query=_GRAPHQL_QUERY, variables=_GraphQLVariables(owner=owner, name=name, number=number, cursor=cursor), ), + read_only=True, ) try: parsed: Final = _GRAPHQL_RESPONSE.validate_python(response.json()) diff --git a/litellm/proxy/roi_calculator/github_observed.py b/litellm/proxy/roi_calculator/github_observed.py new file mode 100644 index 00000000000..a34d34823be --- /dev/null +++ b/litellm/proxy/roi_calculator/github_observed.py @@ -0,0 +1,196 @@ +from datetime import date, datetime, time, timedelta, timezone +from typing import Final, Literal + +import httpx +from pydantic import BaseModel, Field + +from litellm.proxy.roi_calculator.github import GitHubIssueSettings, GitHubPullListItem, SourceError, request_github +from litellm.types.roi_calculator import ROISettings +from litellm.types.roi_observed import ObservedIssue + + +class _PageInfo(BaseModel): + hasNextPage: bool = False + endCursor: str | None = None + + +class _Author(BaseModel): + login: str + kind: str = Field(alias="__typename") + email: str | None = None + + +class _Repository(BaseModel): + nameWithOwner: str + + +class _Label(BaseModel): + name: str + + +class _Labels(BaseModel): + nodes: tuple[_Label, ...] = () + pageInfo: _PageInfo = Field(default_factory=_PageInfo) + + +class _Node(BaseModel): + number: int + url: str + title: str + createdAt: datetime + updatedAt: str + mergedAt: str | None = None + author: _Author | None = None + body: str = "" + headRefName: str = "" + headRefOid: str = "" + headRepository: _Repository | None = None + labels: _Labels = Field(default_factory=_Labels) + + def pull(self) -> GitHubPullListItem: + return GitHubPullListItem.model_validate( + { + "number": self.number, + "html_url": self.url, + "title": self.title, + "body": self.body, + "created_at": self.createdAt, + "merged_at": self.mergedAt, + "updated_at": self.updatedAt, + "user": {"login": self.author.login, "type": self.author.kind, "email": self.author.email} + if self.author + else None, + "head": { + "ref": self.headRefName, + "sha": self.headRefOid, + "repo": {"full_name": self.headRepository.nameWithOwner} if self.headRepository else None, + }, + } + ) + + +class _Search(BaseModel): + issueCount: int + pageInfo: _PageInfo + nodes: tuple[_Node, ...] + + +class _Data(BaseModel): + search: _Search + + +class _Response(BaseModel): + data: _Data | None = None + errors: tuple[object, ...] = () + + +_QUERY: Final = """query($q:String!, $after:String) { + search(query:$q, type:ISSUE, first:100, after:$after) { + issueCount pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { + number url title body createdAt updatedAt mergedAt headRefName headRefOid + author { __typename login ... on User { email } } headRepository { nameWithOwner } + } + ... on Issue { + number url title createdAt updatedAt + labels(first:100) { nodes { name } pageInfo { hasNextPage } } + } + } + } +}""" + + +class GitHubObserved: + def __init__(self, settings: ROISettings, client: httpx.AsyncClient) -> None: + self._client: Final = client + self._api_url: Final = settings.github_api_url + self._url: Final = ( + "https://api.github.com/graphql" + if settings.github_api_url == "https://api.github.com" + else settings.github_api_url.removesuffix("/api/v3") + "/api/graphql" + ) + self._headers: Final = {"Authorization": "Bearer " + settings.github_token.get_secret_value()} + + async def _page(self, query: str, cursor: str | None = None) -> _Search: + response: Final = await request_github( + self._client, + "POST", + self._url, + headers=self._headers, + json_body={"query": _QUERY, "variables": {"q": query, "after": cursor}}, + read_only=True, + ) + try: + result: Final = _Response.model_validate(response.json()) + except ValueError: + raise SourceError("GitHub returned invalid activity data. Try syncing again.") from None + if result.errors or result.data is None: + raise SourceError("GitHub could not read all activity. Check app permissions and rate limits, then retry.") + return result.data.search + + async def _range( + self, repo: str, start: datetime, end: datetime, kind: Literal["pull", "issue"] + ) -> tuple[_Node, ...]: + qualifier: Final = "merged" if kind == "pull" else "created" + source: Final = "is:pr is:merged" if kind == "pull" else "is:issue" + lower: Final = start.strftime("%Y-%m-%dT%H:%M:%SZ") + upper: Final = (end - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + query: Final = f"repo:{repo} {source} {qualifier}:{lower}..{upper} sort:created-asc" + first: Final = await self._page(query) + if first.issueCount > 1000: + seconds: Final = int((end - start).total_seconds()) + if seconds < 2: + raise SourceError("GitHub has more than 1,000 results in one second. The report was not truncated.") + middle: Final = start + timedelta(seconds=seconds // 2) + left: Final = await self._range(repo, start, middle, kind) + return left + await self._range(repo, middle, end, kind) + + async def remaining(page: _Search, seen: frozenset[str]) -> tuple[_Node, ...]: + if not page.pageInfo.hasNextPage: + return page.nodes + cursor: Final = page.pageInfo.endCursor + if not cursor or cursor in seen or len(seen) >= 10: + raise SourceError("GitHub returned incomplete pagination. The previous report was kept.") + following: Final = await self._page(query, cursor) + return page.nodes + await remaining(following, seen | {cursor}) + + nodes: Final = await remaining(first, frozenset()) + if len(nodes) != first.issueCount or len(frozenset(node.url for node in nodes)) != len(nodes): + raise SourceError("GitHub activity changed during collection. Retry to get a complete report.") + return nodes + + async def _read(self, repo: str, start: date, end: date, kind: Literal["pull", "issue"]) -> tuple[_Node, ...]: + return await self._range( + repo, + datetime.combine(start, time.min, timezone.utc), + datetime.combine(end + timedelta(days=1), time.min, timezone.utc), + kind, + ) + + async def pulls(self, repo: str, start: date, end: date) -> tuple[GitHubPullListItem, ...]: + nodes: Final = await self._read(repo, start, end, "pull") + return tuple(node.pull() for node in nodes) + + async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None: + response: Final = await request_github( + self._client, "GET", f"{self._api_url}/repos/{repo}", headers=self._headers + ) + try: + settings: Final = GitHubIssueSettings.model_validate(response.json()) + except ValueError: + raise SourceError("GitHub returned invalid repository settings.") from None + if not settings.has_issues: + return None + nodes: Final = await self._read(repo, start, end, "issue") + if any(node.labels.pageInfo.hasNextPage for node in nodes): + raise SourceError("GitHub returned incomplete issue labels. The previous report was kept.") + return tuple( + ObservedIssue( + repo=repo, + number=node.number, + created_at=node.createdAt, + labels=tuple(label.name for label in node.labels.nodes), + ) + for node in nodes + ) diff --git a/litellm/proxy/roi_calculator/gitlab.py b/litellm/proxy/roi_calculator/gitlab.py index 2df35a0060b..9fbf43d8890 100644 --- a/litellm/proxy/roi_calculator/gitlab.py +++ b/litellm/proxy/roi_calculator/gitlab.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Mapping -from datetime import date +from datetime import date, datetime, timedelta from types import MappingProxyType from typing import Final, TypeVar from urllib.parse import quote @@ -16,6 +16,7 @@ from litellm.proxy.roi_calculator.github import GitHubPullListItem, SourceError from litellm.proxy.roi_calculator.source import repository_tag from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.roi_calculator import ROIPullCommit, ROIPullEvidence, ROIPullFile, ROISettings +from litellm.types.roi_observed import ObservedIssue _T: Final = TypeVar("_T", bound=BaseModel) @@ -23,6 +24,7 @@ _T: Final = TypeVar("_T", bound=BaseModel) class _User(BaseModel): username: str public_email: str | None = None + bot: bool = False class _Project(BaseModel): @@ -30,6 +32,8 @@ class _Project(BaseModel): path_with_namespace: str visibility: str = "private" archived: bool = False + issues_enabled: bool = True + issues_access_level: str = "enabled" class _MergeRequest(BaseModel): @@ -40,6 +44,7 @@ class _MergeRequest(BaseModel): author: _User merged_at: str | None updated_at: str + created_at: datetime | None = None sha: str | None = None source_branch: str source_project_id: int | None @@ -52,7 +57,8 @@ class _MergeRequest(BaseModel): "title": self.title, "body": self.description or "", "html_url": self.web_url, - "user": {"login": self.author.username}, + "user": {"login": self.author.username, "type": "Bot" if self.author.bot else "User"}, + "created_at": self.created_at, "merged_at": self.merged_at, "updated_at": self.updated_at, "head": { @@ -94,11 +100,20 @@ class _Commit(BaseModel): message: str +class _Issue(BaseModel): + iid: int + created_at: datetime + labels: tuple[str, ...] = () + + class GitLab: def __init__(self, settings: ROISettings, transport: httpx.AsyncBaseTransport | None = None) -> None: self.settings: Final = settings token: Final = settings.gitlab_token.get_secret_value() - self.headers: Final = {"Accept": "application/json", **({"PRIVATE-TOKEN": token} if token else {})} + authorization: Final = ( + {"Authorization": f"Bearer {token}"} if settings.connection_type == "app" else {"PRIVATE-TOKEN": token} + ) + self.headers: Final = {"Accept": "application/json", **(authorization if token else {})} self.client: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.ROICalculator, params={"timeout": 45, "follow_redirects": False, "transport": transport}, @@ -171,7 +186,9 @@ class GitLab: params: Final = { "simple": "true", "search": query, - **({"membership": "true"} if self.headers.get("PRIVATE-TOKEN") else {}), + **( + {"membership": "true"} if self.headers.get("PRIVATE-TOKEN") or self.headers.get("Authorization") else {} + ), } items, more = await self._page("projects", _Project, params, page) return tuple((item.path_with_namespace, item.visibility, item.archived) for item in items), more @@ -193,6 +210,8 @@ class GitLab: "state": "merged", "scope": "all", "updated_after": start.isoformat() + "T00:00:00Z", + "merged_after": start.isoformat() + "T00:00:00Z", + "merged_before": (end + timedelta(days=1)).isoformat() + "T00:00:00Z", "order_by": "updated_at", "sort": "desc", }, @@ -218,6 +237,26 @@ class GitLab: self.profiles = MappingProxyType({**self.profiles, login.casefold(): email}) return email + async def issues(self, repo: str, start: date, end: date) -> tuple[ObservedIssue, ...] | None: + project: Final = await self._project(repo) + if not project.issues_enabled or project.issues_access_level == "disabled": + return None + issues: Final = await self._all( + f"projects/{project.id}/issues", + _Issue, + { + "scope": "all", + "state": "all", + "created_after": f"{start}T00:00:00Z", + "created_before": f"{end + timedelta(days=1)}T00:00:00Z", + }, + ) + return tuple( + ObservedIssue(repo=repo, number=issue.iid, created_at=issue.created_at, labels=issue.labels) + for issue in issues + if start <= issue.created_at.date() <= end + ) + async def evidence(self, repo: str, pull: GitHubPullListItem) -> ROIPullEvidence: project: Final = await self._project(repo) path: Final = f"projects/{project.id}/merge_requests/{pull.number}" diff --git a/litellm/proxy/roi_calculator/oauth.py b/litellm/proxy/roi_calculator/oauth.py new file mode 100644 index 00000000000..32d8cd3e1d0 --- /dev/null +++ b/litellm/proxy/roi_calculator/oauth.py @@ -0,0 +1,401 @@ +import asyncio +import hashlib +import json +import os +import re +import secrets +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final, Literal, Protocol, TypeAlias, cast +from urllib.parse import parse_qsl, urlencode, urlsplit + +import httpx +from fastapi import HTTPException +from oauthlib.oauth2 import WebApplicationClient +from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params +) +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.proxy.roi_calculator.settings import ( + active_connection, + connection_id, + load_settings, + load_stored_settings, + save_settings, + select_connection, + stored_connections, +) +from litellm.proxy.roi_calculator.sync_store import SyncStore +from litellm.repositories.config_repository import ConfigRepository +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.roi_calculator import ROISettings, ROISyncStatus + +Provider: TypeAlias = Literal["github", "gitlab"] +_STATE_PREFIX: Final = "roi_oauth_state_" + + +@dataclass(frozen=True, slots=True) +class OAuthConfig: + provider: Provider + api_url: str + base_url: str + client_id: str + client_secret: SecretStr + proxy_url: str + app_slug: str = "" + + @property + def cookie_path(self) -> str: + return urlsplit(self.proxy_url).path + "/roi-calculator/observed/oauth" + + @property + def installation_url(self) -> str | None: + if self.provider != "github" or not self.app_slug: + return None + path: Final = "apps" if self.api_url == "https://api.github.com" else "github-apps" + return f"{self.base_url}/{path}/{self.app_slug}/installations/new" + + @property + def redirect_uri(self) -> str: + return f"{self.proxy_url}/roi-calculator/observed/oauth/{self.provider}/callback" + + @property + def authorize_url(self) -> str: + return self.base_url + ("/login/oauth/authorize" if self.provider == "github" else "/oauth/authorize") + + @property + def token_url(self) -> str: + return self.base_url + ("/login/oauth/access_token" if self.provider == "github" else "/oauth/token") + + +def oauth_config(provider: Provider) -> OAuthConfig | None: + prefix: Final = f"LITELLM_ROI_{provider.upper()}_" + client_id: Final = os.environ.get(prefix + "CLIENT_ID", "") + client_secret: Final = os.environ.get(prefix + "CLIENT_SECRET", "") + proxy_url: Final = os.environ.get("PROXY_BASE_URL", "").rstrip("/") + if not client_id or not client_secret or not proxy_url: + return None + try: + parsed: Final = urlsplit(proxy_url) + except ValueError: + return None + if parsed.username or parsed.password or parsed.query or parsed.fragment or not parsed.hostname: + return None + if parsed.scheme != "https" and not (parsed.scheme == "http" and parsed.hostname in ("localhost", "127.0.0.1")): + return None + base: Final = os.environ.get( + prefix + "URL", "https://github.com" if provider == "github" else "https://gitlab.com" + ).rstrip("/") + api_url: Final = ( + "https://api.github.com" + if base == "https://github.com" + else base + ("/api/v3" if provider == "github" else "/api/v4") + ) + try: + validated: Final = ROISettings.model_validate( + { + "source_provider": provider, + ("github_api_url" if provider == "github" else "gitlab_api_url"): api_url, + } + ) + except ValueError: + return None + app_slug: Final = os.environ.get(prefix + "APP_SLUG", "") + if app_slug and not re.fullmatch(r"[A-Za-z0-9-]+", app_slug): + return None + return OAuthConfig( + provider, validated.source_api_url, base, client_id, SecretStr(client_secret), proxy_url, app_slug + ) + + +class OAuthState(BaseModel): + model_config = ConfigDict(frozen=True) + + provider: Provider + client_id: str + api_url: str + browser_nonce: SecretStr + verifier: SecretStr + expires_at: datetime + settings_revision: int + flow: Literal["authorize", "install"] = "authorize" + + +class _Envelope(BaseModel): + payload: str + + +class _StateRow(BaseModel): + param_value: _Envelope + + +class _Database(Protocol): + async def query_raw(self, query: str, *args: object) -> object: ... + async def execute_raw(self, query: str, *args: object) -> int: ... + + +def _state_key(state: str) -> str: + return _STATE_PREFIX + hashlib.sha256(state.encode()).hexdigest() + + +async def begin_authorization( + repository: ConfigRepository, config: OAuthConfig, *, install: bool = False +) -> tuple[str, str]: + client: Final = WebApplicationClient(config.client_id) + verifier: Final = TypeAdapter(str).validate_python(client.create_code_verifier(64)) + challenge: Final = TypeAdapter(str).validate_python(client.create_code_challenge(verifier, "S256")) + state: Final = secrets.token_urlsafe(32) + nonce: Final = secrets.token_urlsafe(32) + stored: Final = await load_stored_settings(repository) + value: Final = OAuthState( + settings_revision=stored.revision, + flow="install" if install else "authorize", + provider=config.provider, + client_id=config.client_id, + api_url=config.api_url, + browser_nonce=SecretStr(nonce), + verifier=SecretStr(verifier), + expires_at=datetime.now(timezone.utc) + timedelta(minutes=10), + ) + encoded: Final = json.dumps({**value.model_dump(mode="json"), "browser_nonce": nonce, "verifier": verifier}) + payload: Final = TypeAdapter(str).validate_python(encrypt_value_helper(encoded)) + await repository.set_param(_state_key(state), _Envelope(payload=payload).model_dump(mode="json")) + delegate: Final = repository.prisma_client.writer_db + database: Final = cast(_Database, delegate) # cast-ok: Prisma delegates database methods dynamically + await database.execute_raw( + "DELETE FROM \"LiteLLM_Config\" WHERE starts_with(param_name, $1) AND last_run_at < NOW() - INTERVAL '20 minutes'", + _STATE_PREFIX, + ) + url: Final = TypeAdapter(str).validate_python( + client.prepare_request_uri( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped + config.authorize_url, + redirect_uri=config.redirect_uri, + scope="read_api read_user" if config.provider == "gitlab" else None, + state=state, + code_challenge=challenge, + code_challenge_method="S256", + ) + ) + if install and config.installation_url: + return config.installation_url + "?" + urlencode({"state": state}), nonce + return url, nonce + + +async def consume_state( + repository: ConfigRepository, + state: str, + nonce: str, + config: OAuthConfig, + *, + flow: Literal["authorize", "install"] = "authorize", +) -> OAuthState: + if not state or not nonce or len(state) > 200: + raise HTTPException(400, "The connection expired. Start again from the ROI Calculator.") + delegate: Final = repository.prisma_client.writer_db + database: Final = cast(_Database, delegate) # cast-ok: Prisma delegates database methods dynamically + rows: Final = TypeAdapter(tuple[_StateRow, ...]).validate_python( + await database.query_raw( + 'DELETE FROM "LiteLLM_Config" WHERE param_name = $1 RETURNING param_value', + _state_key(state), + ) + ) + if not rows: + raise HTTPException(400, "This connection was already used or expired. Start again.") + plaintext: Final = decrypt_value_helper(rows[0].param_value.payload, _state_key(state)) + if plaintext is None: + raise HTTPException(400, "Could not verify the connection. Start again.") + value: Final = OAuthState.model_validate_json(plaintext) + if ( + value.flow != flow + or not secrets.compare_digest(value.browser_nonce.get_secret_value(), nonce) + or value.expires_at < datetime.now(timezone.utc) + or (value.provider, value.client_id, value.api_url) != (config.provider, config.client_id, config.api_url) + ): + raise HTTPException(400, "Could not verify the connection. Start again in the same browser.") + return value + + +class TokenGrant(BaseModel): + access_token: SecretStr + token_type: str = "bearer" + refresh_token: SecretStr = SecretStr("") + expires_in: int | None = Field(default=None, gt=0) + + +async def _token(config: OAuthConfig, body: str, transport: httpx.AsyncBaseTransport | None) -> TokenGrant: + client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.ROICalculator, + params={"timeout": 30, "follow_redirects": False, "transport": transport}, + ).client + try: + response: Final = await client.post( + config.token_url, data=dict(parse_qsl(body)), headers={"Accept": "application/json"} + ) + except httpx.RequestError: + raise HTTPException(502, "Could not reach the provider. Try connecting again.") from None + finally: + if transport is not None: + await client.aclose() + if response.status_code != 200: + raise HTTPException(502, "The provider rejected the connection. Try connecting again.") + try: + result: Final = TokenGrant.model_validate(response.json()) + except ValueError: + raise HTTPException(502, "The provider did not return a valid token. Try connecting again.") from None + if not result.access_token.get_secret_value() or result.token_type.casefold() != "bearer": + raise HTTPException(502, "The provider returned an unsupported token.") + return result + + +async def exchange_code( + config: OAuthConfig, state: OAuthState, code: str, transport: httpx.AsyncBaseTransport | None = None +) -> TokenGrant: + client: Final = WebApplicationClient(config.client_id) + body: Final = TypeAdapter(str).validate_python( + client.prepare_request_body( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped + code=code, + redirect_uri=config.redirect_uri, + code_verifier=state.verifier.get_secret_value(), + client_secret=config.client_secret.get_secret_value(), + ) + ) + return await _token(config, body, transport) + + +async def save_grant( + repository: ConfigRepository, + config: OAuthConfig, + grant: TokenGrant, + *, + revision: int | None = None, + previous: ROISettings | None = None, + attempt: int = 0, +) -> ROISettings: + stored: Final = await load_stored_settings(repository) + selected: Final = next( + (entry for entry in stored_connections(stored) if entry.id == connection_id(config.provider, config.api_url)), + None, + ) + scoped: Final = select_connection(stored, selected) if selected else stored + current: Final = await load_settings(repository, scoped) + if revision is not None and stored.revision != revision: + raise HTTPException(409, "The connection changed during authorization. Start again from Connections.") + if previous is not None and ( + current.source_provider, + current.source_api_url, + current.connection_type, + current.gitlab_token if current.source_provider == "gitlab" else current.github_token, + current.oauth_refresh_token, + ) != ( + previous.source_provider, + previous.source_api_url, + previous.connection_type, + previous.gitlab_token if previous.source_provider == "gitlab" else previous.github_token, + previous.oauth_refresh_token, + ): + return current + changed: Final = (current.source_provider, current.source_api_url) != (config.provider, config.api_url) + refresh_token: Final = ( + grant.refresh_token + if grant.refresh_token.get_secret_value() or previous is None + else previous.oauth_refresh_token + ) + fields: Final[Mapping[str, object]] = { + "report_mode": "observed" if previous is None else current.report_mode, + "source_provider": config.provider, + "connection_type": "app", + "repos": () if changed else current.repos, + "identity_map": {} if changed else current.identity_map, + "ignored_logins": () if changed else current.ignored_logins, + "oauth_refresh_token": refresh_token, + "oauth_expires_at": datetime.now(timezone.utc) + timedelta(seconds=grant.expires_in) + if grant.expires_in + else None, + ("github_token" if config.provider == "github" else "gitlab_token"): grant.access_token, + ("github_api_url" if config.provider == "github" else "gitlab_api_url"): config.api_url, + } + settings: Final = ROISettings.model_validate({**current.model_dump(), **fields}) + encrypted: Final = TypeAdapter(str).validate_python(encrypt_value_helper(grant.access_token.get_secret_value())) + try: + await save_settings( + repository, + settings, + encrypted if config.provider == "github" else scoped.github_token, + stored.estimator_key, + encrypted if config.provider == "gitlab" else scoped.gitlab_token, + revision=stored.revision, + ) + return settings + except HTTPException as exc: + if exc.status_code != 409 or previous is None or attempt == 2: + raise + + return await save_grant(repository, config, grant, revision=revision, previous=previous, attempt=attempt + 1) + + +def _expired(settings: ROISettings) -> bool: + return bool( + settings.connection_type == "app" + and settings.oauth_expires_at is not None + and settings.oauth_expires_at <= datetime.now(timezone.utc) + timedelta(minutes=5) + ) + + +async def connected_settings( + repository: ConfigRepository, transport: httpx.AsyncBaseTransport | None = None, selected_id: str | None = None +) -> ROISettings: + initial: Final = await load_settings(repository, selected_id=selected_id) + if not _expired(initial): + return initial + selected: Final = selected_id or active_connection(await load_stored_settings(repository)).id + store: Final = SyncStore(repository.prisma_client, "roi_oauth_refresh_" + selected) + owner: Final = secrets.token_urlsafe(24) + status: Final = ROISyncStatus( + running=True, + phase="spend", + stage="Refreshing connection", + done=0, + total=0, + estimated=0, + reused=0, + needs_attention=0, + error=None, + ) + + async def wait_for_connection(attempt: int) -> ROISettings: + if attempt >= 100: + raise HTTPException(409, "The connection is refreshing. Try again shortly.") + settings: Final = await load_settings(repository, selected_id=selected) + if not _expired(settings): + return settings + if not await store.acquire(owner, status): + await asyncio.sleep(0.1) + return await wait_for_connection(attempt + 1) + try: + current: Final = await load_settings(repository, selected_id=selected) + if not _expired(current): + return current + config: Final = oauth_config(current.source_provider) + if ( + config is None + or config.api_url != current.source_api_url + or not current.oauth_refresh_token.get_secret_value() + ): + raise HTTPException(409, "The app connection expired. Reconnect from Connections.") + client: Final = WebApplicationClient(config.client_id) + body: Final = TypeAdapter(str).validate_python( + client.prepare_refresh_body( # pyright: ignore[reportUnknownMemberType] # oauthlib leaves extension kwargs untyped + refresh_token=current.oauth_refresh_token.get_secret_value(), + client_id=config.client_id, + client_secret=config.client_secret.get_secret_value(), + ) + ) + grant: Final = await _token(config, body, transport) + return await save_grant(repository, config, grant, previous=current) + finally: + await store.finish(owner, status.model_copy(update={"running": False, "phase": "complete"})) + + return await wait_for_connection(0) diff --git a/litellm/proxy/roi_calculator/observed_analytics.py b/litellm/proxy/roi_calculator/observed_analytics.py new file mode 100644 index 00000000000..755537aadf0 --- /dev/null +++ b/litellm/proxy/roi_calculator/observed_analytics.py @@ -0,0 +1,191 @@ +import calendar +import re +from collections.abc import Mapping +from datetime import date, datetime, timedelta, timezone +from itertools import chain +from statistics import median +from types import MappingProxyType +from typing import Final + +from litellm.proxy.roi_calculator.analytics import normalize_email +from litellm.proxy.roi_calculator.branch_spend import attribute_branch_keys +from litellm.types.roi_observed import ( + ObservedAccount, + ObservedData, + ObservedHumanSummary, + ObservedPeriod, + ObservedPeriodData, + ObservedPeriods, + ObservedPerson, + ObservedPersonPeriod, + ObservedPersonPeriods, + ObservedPull, + ObservedPullPeriods, + ObservedPullResponse, + ObservedReport, + ObservedWindow, +) + + +def reporting_windows(now: datetime, days: int = 28) -> tuple[ObservedWindow, ObservedWindow, ObservedWindow]: + end: Final = now.astimezone(timezone.utc).date() - timedelta(days=1) + start: Final = end - timedelta(days=days - 1) + last_year_end: Final = date(end.year - 1, end.month, min(end.day, calendar.monthrange(end.year - 1, end.month)[1])) + return ( + ObservedWindow(start=start, end=end), + ObservedWindow(start=start - timedelta(days=days), end=start - timedelta(days=1)), + ObservedWindow(start=last_year_end - timedelta(days=days - 1), end=last_year_end), + ) + + +def declared_requester(author: str, body: str) -> str: + if author.casefold().removesuffix("[bot]") not in ("devin-ai-integration", "devin-ai"): + return "" + matches: Final = frozenset( + match.group(1) for match in re.finditer(r"^Requested by:\s*@([A-Za-z0-9_.-]+)\s*$", body, re.MULTILINE) + ) + return next(iter(matches)).casefold() if len(matches) == 1 else "" + + +def merge_hours(pull: ObservedPull) -> float | None: + if pull.created_at is None or pull.created_at.tzinfo is None or pull.merged_at.tzinfo is None: + return None + seconds: Final = (pull.merged_at - pull.created_at).total_seconds() + return seconds / 3600 if seconds >= 0 else None + + +def median_hours(pulls: tuple[ObservedPull, ...]) -> float | None: + values: Final = tuple(hours for pull in pulls if (hours := merge_hours(pull)) is not None) + return median(values) if values else None + + +def _owner_login(pull: ObservedPull) -> str: + login: Final = (pull.requester if pull.agent else pull.author).casefold() + return f"{pull.connection_id}:{login}" if pull.connection_id and login else login + + +def identity_matches(data: ObservedData, manual: Mapping[str, str], ignored: tuple[str, ...] = ()) -> Mapping[str, str]: + pulls: Final = tuple(chain(data.current.pulls, data.previous.pulls, data.last_year.pulls)) + candidates: Final = frozenset((_owner_login(pull), normalize_email(pull.profile_email)) for pull in pulls) + gateway_emails: Final = frozenset(data.gateway_emails) + profiles: Final = { + login: email + for login, email in candidates + if login not in ignored + and email in gateway_emails + and len({value for key, value in candidates if key == login and value}) == 1 + } + return MappingProxyType({**profiles, **manual}) + + +def _person_period(data: ObservedPeriodData, email: str, identities: Mapping[str, str]) -> ObservedPersonPeriod: + pulls: Final = tuple(pull for pull in data.pulls if identities.get(_owner_login(pull)) == email) + spend: Final = tuple(row for row in data.spend if row["email"] == email) + cost: Final = sum(row["spend"] for row in spend) + days: Final = (data.window.end - data.window.start).days + 1 + return ObservedPersonPeriod( + merged_prs=len(pulls), + prs_per_week=len(pulls) * 7 / days, + median_merge_hours=median_hours(pulls), + direct_authored=sum(not pull.agent for pull in pulls), + declared_agent_owned=sum(pull.agent for pull in pulls), + gateway_recorded_spend=cost, + recorded_spend_per_attributed_pr=cost / len(pulls) if spend and pulls else None, + spend_observation="records_present" if spend else "no_records", + pr_urls=tuple(pull.url for pull in pulls), + ) + + +def _person(data: ObservedData, email: str, identities: Mapping[str, str]) -> ObservedPerson: + return ObservedPerson( + name=email.split("@", 1)[0], + email=email, + logins=tuple( + sorted(frozenset(login.rsplit(":", 1)[-1] for login, address in identities.items() if address == email)) + ), + accounts=tuple( + ObservedAccount( + connection_id=login.split(":", 1)[0] if ":" in login else "", login=login.rsplit(":", 1)[-1] + ) + for login, address in identities.items() + if address == email + ), + periods=ObservedPersonPeriods( + current=_person_period(data.current, email, identities), + previous=_person_period(data.previous, email, identities), + last_year=_person_period(data.last_year, email, identities), + ), + ) + + +def _issue_count(data: ObservedPeriodData, labels: frozenset[str]) -> int | None: + if data.issues is None: + return None + return sum(bool(labels.intersection(label.casefold().strip() for label in issue.labels)) for issue in data.issues) + + +def _period(data: ObservedPeriodData, identities: Mapping[str, str]) -> ObservedPeriod: + matched: Final = tuple(pull for pull in data.pulls if _owner_login(pull) in identities) + emails: Final = frozenset(identities.values()) + spend: Final = tuple(row for row in data.spend if row["email"] in emails) + humans: Final = tuple(pull for pull in data.pulls if pull.author and not pull.agent) + return ObservedPeriod( + window=data.window, + merged_prs=len(data.pulls), + median_merge_hours=median_hours(data.pulls), + human_authored=len(humans), + agent_authored=sum(pull.agent for pull in data.pulls), + missing_author=sum(not pull.author for pull in data.pulls), + agents_without_requester=sum(pull.agent and not pull.requester for pull in data.pulls), + matched_internal_prs=len(matched), + new_bug_labeled_issues=_issue_count(data, frozenset(("bug", "kind:bug", "type::bug"))), + new_regression_labeled_issues=_issue_count( + data, frozenset(("regression", "kind:regression", "type::regression")) + ), + explicitly_titled_revert_prs=sum( + bool(re.match(r"^revert(?:\W|$)", pull.title, re.IGNORECASE)) for pull in data.pulls + ), + matched_users_recorded_spend=sum(row["spend"] for row in spend), + spend_observation="records_present" if spend else "no_records", + human_summary=ObservedHumanSummary(median_merge_hours=median_hours(humans)), + ) + + +def _pulls(data: ObservedPeriodData) -> tuple[ObservedPullResponse, ...]: + costs: Final = attribute_branch_keys( + tuple((pull.url, pull.number, pull.source_repo, pull.source_branch) for pull in data.pulls), data.branch_spend + ) + return tuple( + ObservedPullResponse.model_validate( + {**pull.model_dump(), "merge_hours": merge_hours(pull), "branch_cost": costs[(pull.url, pull.number)]} + ) + for pull in data.pulls + ) + + +def summarize_observed(data: ObservedData, manual: Mapping[str, str], ignored: tuple[str, ...] = ()) -> ObservedReport: + identities: Final = identity_matches(data, manual, ignored) + all_pulls: Final = tuple(chain(data.current.pulls, data.previous.pulls, data.last_year.pulls)) + current_pulls: Final = _pulls(data.current) + linked_branches: Final = frozenset( + (pull.source_repo, pull.source_branch) for pull in current_pulls if pull.branch_cost.status == "matched" + ) + return ObservedReport( + source_provider=data.source_provider, + connections=data.connections, + repos=data.repos, + captured_at=data.captured_at, + periods=ObservedPeriods( + current=_period(data.current, identities), + previous=_period(data.previous, identities), + last_year=_period(data.last_year, identities), + ), + people=tuple(_person(data, email, identities) for email in sorted(frozenset(identities.values()))), + pulls=ObservedPullPeriods( + current=current_pulls, previous=_pulls(data.previous), last_year=_pulls(data.last_year) + ), + unlinked_branches=tuple( + row for row in data.current.branch_spend or () if (row.repo, row.branch) not in linked_branches + ), + unmatched_logins=tuple(sorted(frozenset(_owner_login(pull) for pull in all_pulls) - identities.keys() - {""})), + ) diff --git a/litellm/proxy/roi_calculator/observed_sync.py b/litellm/proxy/roi_calculator/observed_sync.py new file mode 100644 index 00000000000..c2c5ae0e31c --- /dev/null +++ b/litellm/proxy/roi_calculator/observed_sync.py @@ -0,0 +1,265 @@ +import asyncio +from collections.abc import Awaitable, Callable, Mapping +from contextlib import suppress +from datetime import datetime, timezone +from itertools import chain +from types import MappingProxyType +from typing import Final, TypeAlias +from uuid import uuid4 + +import httpx + +from litellm._logging import verbose_proxy_logger +from litellm.proxy.roi_calculator.analytics import normalize_email +from litellm.proxy.roi_calculator.github import GitHub, GitHubPullListItem, SourceError +from litellm.proxy.roi_calculator.github_observed import GitHubObserved +from litellm.proxy.roi_calculator.gitlab import GitLab +from litellm.proxy.roi_calculator.observed_analytics import declared_requester, reporting_windows +from litellm.proxy.roi_calculator.source import repository_tag +from litellm.proxy.roi_calculator.sync import BranchSpendReader, GatewayUserReader, SpendReader +from litellm.proxy.roi_calculator.sync_store import SyncStore +from litellm.types.roi_calculator import ROISettings, ROISyncStatus +from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow + + +def _author(pull: GitHubPullListItem) -> str: + return pull.user.login or "" if pull.user else "" + + +def _agent(pull: GitHubPullListItem) -> bool: + login: Final = _author(pull) + return ( + bool(pull.user and pull.user.type == "Bot") + or login.endswith("[bot]") + or bool(login.startswith(("project_", "group_")) and "_bot_" in login) + ) + + +def _owner(pull: GitHubPullListItem) -> str: + login: Final = _author(pull) + return declared_requester(login, pull.body or "") if _agent(pull) else login + + +def _public_email(pull: GitHubPullListItem) -> str: + return normalize_email(pull.user.email) if pull.user and not _agent(pull) else "" + + +def _pull(settings: ROISettings, repo: str, pull: GitHubPullListItem, profiles: Mapping[str, str]) -> ObservedPull: + if not pull.merged_at: + raise SourceError("The source returned an unmerged change. No partial report was saved.") + return ObservedPull( + repo=repo, + number=pull.number, + title=pull.title, + url=pull.html_url, + author=_author(pull), + agent=_agent(pull), + requester=declared_requester(_author(pull), pull.body or "") if _agent(pull) else "", + profile_email=profiles.get(_owner(pull).casefold(), "") or _public_email(pull), + created_at=pull.created_at, + merged_at=datetime.fromisoformat(pull.merged_at.replace("Z", "+00:00")), + source_repo=repository_tag(settings, pull.head.repo.full_name) if pull.head and pull.head.repo else "", + source_branch=pull.head.ref if pull.head else "", + ) + + +async def collect_observed( + settings: ROISettings, + spend_reader: SpendReader, + gateway_user_reader: GatewayUserReader, + branch_spend_reader: BranchSpendReader, + now: datetime, + progress: Callable[[str, int, int], None], + transport: httpx.AsyncBaseTransport | None = None, + days: int = 28, +) -> ObservedData: + source: Final = GitLab(settings, transport) if settings.source_provider == "gitlab" else GitHub(settings, transport) + activity: Final = ( + GitHubObserved(settings, source.client) + if settings.source_provider == "github" and settings.github_token.get_secret_value() + else source + ) + windows: Final = reporting_windows(now.astimezone(timezone.utc), days) + slots: Final = asyncio.Semaphore(4) + total: Final = len(settings.repos) * 3 + + async def profile(login: str) -> tuple[str, str]: + async with slots: + return login.casefold(), await source.profile_email(login) + + async def repository( + repo: str, window: ObservedWindow + ) -> tuple[tuple[ObservedPull, ...], tuple[ObservedIssue, ...] | None]: + raw: Final = await activity.pulls(repo, window.start, window.end) + unique: Final = {(repo, item.number): item for item in raw} + if len(unique) != len(raw): + raise SourceError("The source returned duplicate changes. Retry to get a complete report.") + owners: Final = ( + frozenset(_owner(pull) for pull in raw if not _public_email(pull)) + - {""} + - settings.identity_map.keys() + - frozenset(settings.ignored_logins) + ) + profiles: Final = MappingProxyType(dict(await asyncio.gather(*(profile(login) for login in owners)))) + pulls: Final = tuple(_pull(settings, repo, item, profiles) for item in raw) + issues: Final = await activity.issues(repo, window.start, window.end) + return pulls, issues + + async def period(window: ObservedWindow, offset: int) -> ObservedPeriodData: + async def read(index: int, repo: str) -> tuple[tuple[ObservedPull, ...], tuple[ObservedIssue, ...] | None]: + progress(f"Reading {repo} ({window.start} to {window.end})", offset + index, total) + return await repository(repo, window) + + results: Final = tuple([await read(index, repo) for index, repo in enumerate(settings.repos)]) + pulls: Final = tuple(chain.from_iterable(result[0] for result in results)) + issues: Final = ( + None + if results and all(result[1] is None for result in results) + else tuple(chain.from_iterable(result[1] or () for result in results)) + ) + branches: Final = tuple( + sorted( + frozenset( + ( + *(repository_tag(settings, repo) for repo in settings.repos), + *(pull.source_repo for pull in pulls), + ) + ) + - {""} + ) + ) + return ObservedPeriodData( + window=window, + pulls=tuple(sorted(pulls, key=lambda pull: (pull.merged_at, pull.repo, pull.number), reverse=True)), + issues=issues, + spend=await spend_reader(window.start, window.end), + branch_spend=await branch_spend_reader(window.start, window.end, branches), + ) + + try: + gateway_emails: Final = await gateway_user_reader() + current: Final = await period(windows[0], 0) + previous: Final = await period(windows[1], len(settings.repos)) + last_year: Final = await period(windows[2], len(settings.repos) * 2) + progress("Saving report", total, total) + return ObservedData( + source_provider=settings.source_provider, + source_api_url=settings.source_api_url, + repos=settings.repos, + captured_at=now, + gateway_emails=tuple(sorted(gateway_emails)), + current=current, + previous=previous, + last_year=last_year, + ) + finally: + await source.close() + + +Progress: TypeAlias = Callable[[str, int, int], None] +BuildReport: TypeAlias = Callable[[Progress], Awaitable[ObservedData]] + + +class ObservedSyncManager: + def __init__(self) -> None: + self.status: ROISyncStatus = ROISyncStatus( + running=False, + phase="idle", + stage="Not synced", + done=0, + total=0, + estimated=0, + reused=0, + needs_attention=0, + error=None, + ) + self._task: asyncio.Task[None] | None = None + self._lock: Final = asyncio.Lock() + + def _progress(self, stage: str, done: int, total: int) -> None: + self.status = self.status.model_copy( + update={"phase": "repositories", "stage": stage, "done": done, "total": total} + ) + + async def start(self, build: BuildReport, store: SyncStore, scheduled_interval: float = 0) -> bool: + async with self._lock: + if self._task is not None and not self._task.done(): + return False + status: Final = ROISyncStatus( + running=True, + phase="repositories", + stage="Reading repository activity", + done=0, + total=0, + estimated=0, + reused=0, + needs_attention=0, + error=None, + started_at=datetime.now(timezone.utc).isoformat(), + ) + owner: Final = str(uuid4()) + if not await store.acquire(owner, status, scheduled_interval): + return False + self.status = status + self._task = asyncio.create_task(self._run(build, store, owner)) + return True + + async def cancel(self) -> None: + if self._task is not None and not self._task.done(): + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + + async def _heartbeat(self, store: SyncStore, owner: str, task: asyncio.Task[object] | None) -> None: + if task is None: + return + try: + while True: + await asyncio.sleep(5) + if not await store.heartbeat(owner, self.status): + task.cancel() + return + except Exception: # noqa: BLE001 # loss of the database lease must stop publication + task.cancel() + + async def _run(self, build: BuildReport, store: SyncStore, owner: str) -> None: + monitor: Final = asyncio.create_task(self._heartbeat(store, owner, asyncio.current_task())) + try: + report: Final = await build(self._progress) + monitor.cancel() + with suppress(asyncio.CancelledError): + await monitor + complete: Final = self.status.model_copy( + update={ + "running": False, + "phase": "complete", + "stage": "Up to date", + "finished_at": datetime.now(timezone.utc).isoformat(), + } + ) + if not await store.finish(owner, complete, report): + raise SourceError("The sync was cancelled or replaced. The previous report was kept.") + self.status = complete + except asyncio.CancelledError: + self.status = self.status.model_copy(update={"phase": "cancelled", "stage": "Sync cancelled"}) + raise + except SourceError as exc: + self.status = self.status.model_copy(update={"phase": "error", "stage": "Sync failed", "error": str(exc)}) + except Exception: # noqa: BLE001 # background tasks must persist a safe error without exposing credentials + verbose_proxy_logger.exception("Observed ROI sync failed") + self.status = self.status.model_copy( + update={ + "phase": "error", + "stage": "Sync failed", + "error": "Could not finish syncing. The previous report was kept. Retry after checking the connection.", + } + ) + finally: + monitor.cancel() + with suppress(asyncio.CancelledError): + await monitor + self.status = self.status.model_copy( + update={"running": False, "finished_at": datetime.now(timezone.utc).isoformat()} + ) + if self.status.phase != "complete": + await store.finish(owner, self.status) diff --git a/litellm/proxy/roi_calculator/observed_workspace.py b/litellm/proxy/roi_calculator/observed_workspace.py new file mode 100644 index 00000000000..f08e0f5d681 --- /dev/null +++ b/litellm/proxy/roi_calculator/observed_workspace.py @@ -0,0 +1,135 @@ +from collections.abc import Mapping +from datetime import date, datetime +from itertools import chain +from types import MappingProxyType +from typing import Final + +import httpx + +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.observed_analytics import reporting_windows, summarize_observed +from litellm.proxy.roi_calculator.observed_sync import Progress, collect_observed +from litellm.proxy.roi_calculator.settings import StoredConnection, connection_id +from litellm.proxy.roi_calculator.source import repository_tag +from litellm.proxy.roi_calculator.sync import BranchSpendReader, GatewayUserReader, SpendReader +from litellm.types.roi_calculator import ROISettings, ROISpendRecord +from litellm.types.roi_observed import ObservedData, ObservedPeriodData, ObservedReport, ObservedSource + + +def source_details(settings: ROISettings) -> ObservedSource: + return ObservedSource( + id=connection_id(settings.source_provider, settings.source_api_url), + source_provider=settings.source_provider, + api_url=settings.source_api_url, + repos=settings.repos, + ) + + +def scoped_data(data: ObservedData, source: ObservedSource) -> ObservedData: + def period(value: ObservedPeriodData) -> ObservedPeriodData: + return value.model_copy( + update={"pulls": tuple(pull.model_copy(update={"connection_id": source.id}) for pull in value.pulls)} + ) + + return data.model_copy( + update={ + "connections": (source,), + "current": period(data.current), + "previous": period(data.previous), + "last_year": period(data.last_year), + } + ) + + +def summarize_workspace(data: ObservedData, connections: tuple[StoredConnection, ...]) -> ObservedReport: + included: Final = frozenset(source.id for source in data.connections) + active: Final = tuple(entry for entry in connections if entry.id in included) + maps: Final = ({f"{entry.id}:{login}": email for login, email in entry.identity_map.items()} for entry in active) + identities: Final = MappingProxyType(dict(chain.from_iterable(mapping.items() for mapping in maps))) + ignored: Final = tuple( + chain.from_iterable(tuple(f"{entry.id}:{login}" for login in entry.ignored_logins) for entry in connections) + ) + return summarize_observed(data, identities, ignored) + + +def combine_observed(sources: tuple[ObservedData, ...], repos: tuple[str, ...]) -> ObservedData: + first: Final = sources[0] + + def period(values: tuple[ObservedPeriodData, ...]) -> ObservedPeriodData: + if any(value.window != values[0].window or value.spend != values[0].spend for value in values): + raise SourceError("The reporting windows changed during sync. Retry to get a complete report.") + pulls: Final = tuple(chain.from_iterable(value.pulls for value in values)) + if len({pull.url for pull in pulls}) != len(pulls): + raise SourceError("A repository is selected through more than one connection. Select it once.") + return ObservedPeriodData( + window=values[0].window, + pulls=tuple(sorted(pulls, key=lambda pull: (pull.merged_at, pull.url), reverse=True)), + issues=None + if all(value.issues is None for value in values) + else tuple(chain.from_iterable(value.issues or () for value in values)), + spend=values[0].spend, + branch_spend=None + if any(value.branch_spend is None for value in values) + else tuple( + { + (row.repo, row.branch): row + for row in chain.from_iterable(value.branch_spend or () for value in values) + }.values() + ), + ) + + providers: Final = frozenset(source.source_provider for source in sources) + return ObservedData( + source_provider=first.source_provider if len(providers) == 1 else "mixed", + source_api_url=first.source_api_url if len(sources) == 1 else "", + connections=tuple(chain.from_iterable(source.connections for source in sources)), + repos=repos, + captured_at=first.captured_at, + gateway_emails=first.gateway_emails, + current=period(tuple(source.current for source in sources)), + previous=period(tuple(source.previous for source in sources)), + last_year=period(tuple(source.last_year for source in sources)), + ) + + +async def collect_workspace( + connections: tuple[tuple[ROISettings, BranchSpendReader], ...], + spend_reader: SpendReader, + gateway_user_reader: GatewayUserReader, + now: datetime, + progress: Progress, + transport: httpx.AsyncBaseTransport | None = None, + days: int = 28, +) -> ObservedData: + windows: Final = reporting_windows(now, days) + spending: Final[Mapping[tuple[date, date], tuple[ROISpendRecord, ...]]] = { + (window.start, window.end): await spend_reader(window.start, window.end) for window in windows + } + emails: Final = await gateway_user_reader() + total: Final = sum(len(settings.repos) * 3 for settings, _reader in connections) + + async def spend(start: date, end: date) -> tuple[ROISpendRecord, ...]: + return spending[(start, end)] + + async def users() -> frozenset[str]: + return emails + + async def collect(index: int, settings: ROISettings, branch_reader: BranchSpendReader) -> ObservedData: + offset: Final = sum(len(prior.repos) * 3 for prior, _reader in connections[:index]) + + def update(stage: str, done: int, _total: int) -> None: + progress(stage, offset + done, total) + + data: Final = await collect_observed(settings, spend, users, branch_reader, now, update, transport, days=days) + return scoped_data(data, source_details(settings)) + + data: Final = tuple( + [await collect(index, settings, reader) for index, (settings, reader) in enumerate(connections)] + ) + repos: Final = tuple( + chain.from_iterable( + tuple(repository_tag(settings, repo) if len(connections) > 1 else repo for repo in settings.repos) + for settings, _reader in connections + ) + ) + return combine_observed(data, repos) diff --git a/litellm/proxy/roi_calculator/settings.py b/litellm/proxy/roi_calculator/settings.py new file mode 100644 index 00000000000..89a691843f4 --- /dev/null +++ b/litellm/proxy/roi_calculator/settings.py @@ -0,0 +1,253 @@ +from collections.abc import Mapping +from datetime import datetime +from hashlib import sha256 +from types import MappingProxyType +from typing import Annotated, Final, Literal + +from fastapi import Depends, HTTPException +from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError + +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.repositories.config_repository import ConfigRepository +from litellm.types.roi_calculator import DEFAULT_PROMPT, ROISettings + +_SETTINGS_KEY: Final = "roi_calculator_settings" + + +def connection_id(provider: str, api_url: str) -> str: + return provider + "_" + sha256(api_url.strip().rstrip("/").encode()).hexdigest()[:16] + + +class StoredConnection(BaseModel): + model_config = ConfigDict(frozen=True) + + source_provider: Literal["github", "gitlab"] + api_url: str + token: str = "" + connection_type: Literal["token", "app"] = "token" + oauth_refresh_token: str = "" + oauth_expires_at: datetime | None = None + repos: tuple[str, ...] = () + identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({})) + ignored_logins: tuple[str, ...] = () + + @property + def id(self) -> str: + return connection_id(self.source_provider, self.api_url) + + +class StoredROISettings(BaseModel): + model_config = ConfigDict(extra="ignore") + + revision: int = 0 + report_mode: Literal["legacy", "observed"] = "legacy" + source_provider: Literal["github", "gitlab"] = "github" + connection_type: Literal["token", "app"] = "token" + oauth_refresh_token: str = "" + oauth_expires_at: datetime | None = None + ignored_logins: tuple[str, ...] = () + gitlab_api_url: str = "https://gitlab.com/api/v4" + gitlab_token: str = "" + github_api_url: str = "https://api.github.com" + github_token: str = "" + estimator_key: str = "" + repos: tuple[str, ...] = () + estimator_model: str = "" + estimator_prompt: str = DEFAULT_PROMPT + backfill_days: int = Field(default=7, ge=1, le=3650) + update_interval_minutes: float = Field(default=1440, ge=0, le=43200) + identity_map: Mapping[str, str] = Field(default_factory=lambda: MappingProxyType({})) + connections: tuple[StoredConnection, ...] = () + + +def active_connection(stored: StoredROISettings) -> StoredConnection: + return StoredConnection( + source_provider=stored.source_provider, + api_url=stored.gitlab_api_url if stored.source_provider == "gitlab" else stored.github_api_url, + token=stored.gitlab_token if stored.source_provider == "gitlab" else stored.github_token, + connection_type=stored.connection_type, + oauth_refresh_token=stored.oauth_refresh_token, + oauth_expires_at=stored.oauth_expires_at, + repos=stored.repos, + identity_map=stored.identity_map, + ignored_logins=stored.ignored_logins, + ) + + +def stored_connections(stored: StoredROISettings) -> tuple[StoredConnection, ...]: + active: Final = active_connection(stored) + if not stored.connections and not active.repos and not active.token: + return () + return tuple({**{entry.id: entry for entry in stored.connections}, active.id: active}.values()) + + +def select_connection(stored: StoredROISettings, selected: StoredConnection) -> StoredROISettings: + return stored.model_copy( + update={ + "source_provider": selected.source_provider, + "connection_type": selected.connection_type, + "oauth_refresh_token": selected.oauth_refresh_token, + "oauth_expires_at": selected.oauth_expires_at, + "repos": selected.repos, + "identity_map": selected.identity_map, + "ignored_logins": selected.ignored_logins, + ("gitlab_api_url" if selected.source_provider == "gitlab" else "github_api_url"): selected.api_url, + ("gitlab_token" if selected.source_provider == "gitlab" else "github_token"): selected.token, + } + ) + + +async def read_admin( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> UserAPIKeyAuth: + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException(status_code=403, detail="Only proxy admins can access the ROI Calculator.") + return user_api_key_dict + + +async def write_admin( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> UserAPIKeyAuth: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Only proxy admins can change ROI Calculator settings.") + return user_api_key_dict + + +async def get_roi_config_repository( + _user: Annotated[UserAPIKeyAuth, Depends(read_admin)], +) -> ConfigRepository: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail=CommonProxyErrors.db_not_connected_error.value, + ) + return ConfigRepository(prisma_client, use_writer=True) + + +async def load_stored_settings(repository: ConfigRepository, selected_id: str | None = None) -> StoredROISettings: + parameter: Final = await repository.get_param(_SETTINGS_KEY) + if parameter is None: + if selected_id is not None: + raise HTTPException(404, "This connection no longer exists. Reload Connections.") + return StoredROISettings() + try: + stored: Final = StoredROISettings.model_validate(parameter.param_value) + except ValidationError: + raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None + if selected_id is None: + return stored + selected: Final = next((entry for entry in stored_connections(stored) if entry.id == selected_id), None) + if selected is None: + raise HTTPException(404, "This connection no longer exists. Reload Connections.") + return select_connection(stored, selected) + + +async def load_settings( + repository: ConfigRepository, value: StoredROISettings | None = None, selected_id: str | None = None +) -> ROISettings: + stored: Final = value if value is not None else await load_stored_settings(repository, selected_id) + token: Final = decrypt_value_helper(stored.github_token, _SETTINGS_KEY) if stored.github_token else "" + try: + return ROISettings( + report_mode=stored.report_mode, + source_provider=stored.source_provider, + connection_type=stored.connection_type, + oauth_refresh_token=SecretStr(decrypt_value_helper(stored.oauth_refresh_token, _SETTINGS_KEY) or "") + if stored.oauth_refresh_token + else SecretStr(""), + oauth_expires_at=stored.oauth_expires_at, + ignored_logins=stored.ignored_logins, + gitlab_api_url=stored.gitlab_api_url, + gitlab_token=SecretStr(decrypt_value_helper(stored.gitlab_token, _SETTINGS_KEY) or "") + if stored.gitlab_token + else SecretStr(""), + github_api_url=stored.github_api_url, + github_token=SecretStr(token or ""), + estimator_key=SecretStr(decrypt_value_helper(stored.estimator_key, _SETTINGS_KEY) or "") + if stored.estimator_key + else SecretStr(""), + update_interval_minutes=stored.update_interval_minutes, + repos=stored.repos, + estimator_model=stored.estimator_model, + estimator_prompt=stored.estimator_prompt, + backfill_days=stored.backfill_days, + identity_map=stored.identity_map, + ) + except ValidationError: + raise HTTPException(status_code=500, detail="Stored ROI Calculator settings are invalid.") from None + + +async def save_settings( + repository: ConfigRepository, + settings: ROISettings, + encrypted_token: str, + encrypted_estimator_key: str, + encrypted_gitlab_token: str = "", + revision: int = 0, + replace_connection_id: str | None = None, +) -> None: + previous: Final = await load_stored_settings(repository) + stored: Final = StoredROISettings( + revision=revision + 1, + report_mode=settings.report_mode, + source_provider=settings.source_provider, + connection_type=settings.connection_type, + oauth_refresh_token=TypeAdapter(str).validate_python( + encrypt_value_helper(settings.oauth_refresh_token.get_secret_value()) + ) + if settings.oauth_refresh_token.get_secret_value() + else "", + oauth_expires_at=settings.oauth_expires_at, + ignored_logins=settings.ignored_logins, + gitlab_api_url=settings.gitlab_api_url, + gitlab_token=encrypted_gitlab_token, + github_api_url=settings.github_api_url, + github_token=encrypted_token, + estimator_key=encrypted_estimator_key, + update_interval_minutes=settings.update_interval_minutes, + repos=settings.repos, + estimator_model=settings.estimator_model, + estimator_prompt=settings.estimator_prompt, + backfill_days=settings.backfill_days, + identity_map=settings.identity_map, + ) + active: Final = active_connection(stored) + combined: Final = stored.model_copy( + update={ + "connections": tuple( + { + **{entry.id: entry for entry in stored_connections(previous) if entry.id != replace_connection_id}, + active.id: active, + }.values() + ) + } + ) + if not await repository.set_param_if_revision(_SETTINGS_KEY, combined.model_dump(mode="json"), revision): + raise HTTPException(409, "Settings changed while you were editing. Reload and try again.") + + +async def enable_observed_reporting(repository: ConfigRepository) -> None: + stored: Final = await load_stored_settings(repository) + if stored.report_mode == "observed": + return + updated: Final = stored.model_copy(update={"report_mode": "observed", "revision": stored.revision + 1}) + if not await repository.set_param_if_revision(_SETTINGS_KEY, updated.model_dump(mode="json"), stored.revision): + raise HTTPException(409, "Settings changed while starting the report. Reload and try again.") + + +async def save_connection_identities( + repository: ConfigRepository, stored: StoredROISettings, connections: tuple[StoredConnection, ...] +) -> None: + active: Final = next(entry for entry in connections if entry.id == active_connection(stored).id) + updated: Final = select_connection(stored, active).model_copy( + update={"connections": connections, "revision": stored.revision + 1} + ) + if not await repository.set_param_if_revision(_SETTINGS_KEY, updated.model_dump(mode="json"), stored.revision): + raise HTTPException(409, "Settings changed while you were editing. Reload and try again.") diff --git a/litellm/proxy/roi_calculator/sync_store.py b/litellm/proxy/roi_calculator/sync_store.py index 43a2533eb59..df126da6d70 100644 --- a/litellm/proxy/roi_calculator/sync_store.py +++ b/litellm/proxy/roi_calculator/sync_store.py @@ -1,11 +1,12 @@ from datetime import datetime, timezone from types import MappingProxyType -from typing import Final, Protocol, cast # noqa: TID251 - PrismaWrapper dynamically delegates database methods +from typing import Final, Literal, Protocol, cast # noqa: TID251 - PrismaWrapper dynamically delegates database methods from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm.proxy.utils import PrismaClient from litellm.types.roi_calculator import ROIReport, ROISyncStatus +from litellm.types.roi_observed import ObservedData _SYNC_KEY: Final = "roi_calculator_sync" _REPORT_KEY: Final = "roi_calculator_report" @@ -30,8 +31,14 @@ class _SyncDatabase(Protocol): class SyncStore: - def __init__(self, prisma: PrismaClient) -> None: + def __init__( + self, + prisma: PrismaClient, + namespace: Literal["roi_calculator", "roi_observed", "roi_oauth_refresh"] = "roi_calculator", + ) -> None: self._db: Final = cast(_SyncDatabase, prisma.writer_db) # cast-ok: PrismaWrapper delegates methods dynamically + self._sync_key: Final = namespace + "_sync" + self._report_key: Final = namespace + "_report" async def acquire(self, owner: str, status: ROISyncStatus, scheduled_interval: float = 0) -> bool: rows: Final = await self._db.query_raw( @@ -43,7 +50,7 @@ class SyncStore: OR "LiteLLM_Config".param_value->'status'->>'running' = 'false') AND ($3::text::double precision = 0 OR "LiteLLM_Config".last_run_at <= NOW() - $3::text::double precision * INTERVAL '1 minute') RETURNING param_name""", - _SYNC_KEY, + self._sync_key, _SyncState(owner=owner, status=status).model_dump_json(), str(scheduled_interval), ) @@ -58,14 +65,20 @@ class SyncStore: AND param_value->'status'->>'running' = 'true' AND last_run_at >= NOW() - INTERVAL '60 seconds' RETURNING param_name""", - _SYNC_KEY, + self._sync_key, owner, status.model_dump_json(), ) return bool(rows) - async def finish(self, owner: str, status: ROISyncStatus, report: ROIReport | None = None) -> bool: - report_json: Final = TypeAdapter(ROIReport).dump_json(report).decode() if report is not None else None + async def finish(self, owner: str, status: ROISyncStatus, report: ROIReport | ObservedData | None = None) -> bool: + report_json: Final = ( + report.model_dump_json() + if isinstance(report, ObservedData) + else TypeAdapter(ROIReport).dump_json(report).decode() + if report is not None + else None + ) rows: Final = await self._db.query_raw( """WITH owned AS ( SELECT param_name FROM "LiteLLM_Config" @@ -92,11 +105,11 @@ class SyncStore: UPDATE "LiteLLM_Config" SET param_value = jsonb_set(param_value, '{status}', $3::jsonb), last_run_at = NOW() WHERE param_name IN (SELECT param_name FROM owned) RETURNING param_name""", - _SYNC_KEY, + self._sync_key, owner, status.model_dump_json(), report_json, - _REPORT_KEY, + self._report_key, ) return bool(rows) @@ -105,7 +118,7 @@ class SyncStore: await self._db.query_raw( """SELECT param_value, last_run_at, last_run_at < NOW() - INTERVAL '60 seconds' AS expired FROM "LiteLLM_Config" WHERE param_name = $1""", - _SYNC_KEY, + self._sync_key, ) ) if not rows: @@ -119,7 +132,7 @@ class SyncStore: "phase": "error", "finished_at": rows[0].last_run_at.replace(tzinfo=timezone.utc).isoformat(), "stage": "Sync interrupted", - "error": "The worker stopped responding. Run analysis again to resume saved estimates.", + "error": "The worker stopped responding. Sync again to refresh the report.", } ) ) @@ -136,8 +149,8 @@ class SyncStore: ) ), last_run_at = NOW() WHERE param_name = $1 AND param_value->'status'->>'running' = 'true' """, - _SYNC_KEY, + self._sync_key, ) async def clear_report(self) -> None: - await self._db.execute_raw('DELETE FROM "LiteLLM_Config" WHERE param_name = $1', _REPORT_KEY) + await self._db.execute_raw('DELETE FROM "LiteLLM_Config" WHERE param_name = $1', self._report_key) diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index c5674a4b398..c0be8ac55a9 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -33,6 +33,10 @@ class _ConfigTable(Protocol): async def delete(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ... +class _ConfigDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> object: ... + + class ConfigParam: """Simple wrapper for config parameter from DB.""" @@ -85,6 +89,26 @@ class ConfigRepository: ) return ConfigParam(param_name=param_name, param_value=param_value) + async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool: + delegate: Final = self.prisma_client.writer_db + database: Final = cast(_ConfigDatabase, delegate) # cast-ok: Prisma delegates database methods dynamically + rows: Final = await database.query_raw( + """INSERT INTO "LiteLLM_Config" (param_name, param_value, last_run_at) + SELECT $1, $2::jsonb, NOW() WHERE $3::int = 0 + ON CONFLICT (param_name) DO UPDATE + SET param_value = EXCLUDED.param_value, last_run_at = NOW() + WHERE COALESCE(("LiteLLM_Config".param_value->>'revision')::int, 0) = $3::int + RETURNING param_name""" + if revision == 0 + else """UPDATE "LiteLLM_Config" SET param_value = $2::jsonb, last_run_at = NOW() + WHERE param_name = $1 AND COALESCE((param_value->>'revision')::int, 0) = $3::int + RETURNING param_name""", + param_name, + json.dumps(param_value), + revision, + ) + return bool(rows) + async def delete_param(self, param_name: str) -> bool: """Delete a config parameter from the database.""" try: diff --git a/litellm/types/roi_calculator.py b/litellm/types/roi_calculator.py index 63a28ec71ca..7a6cd8ab2e1 100644 --- a/litellm/types/roi_calculator.py +++ b/litellm/types/roi_calculator.py @@ -1,4 +1,5 @@ from collections.abc import Mapping +from datetime import datetime from types import MappingProxyType from typing import Final, Literal @@ -24,7 +25,12 @@ def normalize_source_login(value: str, provider: str = "github") -> str: class ROISettings(BaseModel): model_config = ConfigDict(frozen=True) + report_mode: Literal["legacy", "observed"] = "legacy" source_provider: Literal["github", "gitlab"] = "github" + connection_type: Literal["token", "app"] = "token" + oauth_refresh_token: SecretStr = SecretStr("") + oauth_expires_at: datetime | None = None + ignored_logins: tuple[str, ...] = () gitlab_api_url: str = "https://gitlab.com/api/v4" gitlab_token: SecretStr = SecretStr("") github_api_url: str = "https://api.github.com" @@ -74,8 +80,13 @@ class ROISettings(BaseModel): import re normalized_values: Final = tuple(repo.strip().rstrip("/").removesuffix(".git") for repo in values) + repository_keys: Final = tuple( + repo.casefold() if info.data.get("source_provider") != "gitlab" else repo for repo in normalized_values + ) normalized: Final = tuple( - repo for index, repo in enumerate(normalized_values) if repo not in normalized_values[:index] + repo + for index, repo in enumerate(normalized_values) + if repository_keys[index] not in repository_keys[:index] ) pattern: Final = ( r"[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+" @@ -121,6 +132,7 @@ class ROISettings(BaseModel): class ROISettingsUpdate(BaseModel): model_config = ConfigDict(extra="forbid") + report_mode: Literal["legacy", "observed"] | None = None source_provider: Literal["github", "gitlab"] | None = None gitlab_api_url: str | None = None gitlab_token: str | None = None @@ -140,6 +152,7 @@ class ROIEstimatorModel(BaseModel): class ROISettingsResponse(BaseModel): + report_mode: Literal["legacy", "observed"] = "legacy" source_provider: Literal["github", "gitlab"] = "github" gitlab_api_url: str = "https://gitlab.com/api/v4" has_gitlab_token: bool = False diff --git a/litellm/types/roi_observed.py b/litellm/types/roi_observed.py new file mode 100644 index 00000000000..b617175788b --- /dev/null +++ b/litellm/types/roi_observed.py @@ -0,0 +1,213 @@ +from collections.abc import Mapping +from datetime import date, datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from litellm.types.roi_calculator import ROIBranchAttribution, ROIBranchSpend, ROISpendRecord + + +class ObservedModel(BaseModel): + model_config = ConfigDict(frozen=True) + + +class ObservedWindow(ObservedModel): + start: date + end: date + + +class ObservedSource(ObservedModel): + id: str + source_provider: Literal["github", "gitlab"] + api_url: str + repos: tuple[str, ...] + + +class ObservedAccount(ObservedModel): + connection_id: str + login: str + + +class ObservedPull(ObservedModel): + connection_id: str = "" + repo: str + number: int + title: str + url: str + author: str + agent: bool = False + requester: str = "" + profile_email: str = "" + created_at: datetime | None = None + merged_at: datetime + source_repo: str = "" + source_branch: str = "" + + +class ObservedIssue(ObservedModel): + repo: str + number: int + created_at: datetime + labels: tuple[str, ...] = () + + +class ObservedPeriodData(ObservedModel): + window: ObservedWindow + pulls: tuple[ObservedPull, ...] + issues: tuple[ObservedIssue, ...] | None + spend: tuple[ROISpendRecord, ...] + branch_spend: tuple[ROIBranchSpend, ...] | None = None + + +class ObservedData(ObservedModel): + source_provider: Literal["github", "gitlab", "mixed"] + connections: tuple[ObservedSource, ...] = () + source_api_url: str + repos: tuple[str, ...] + captured_at: datetime + gateway_emails: tuple[str, ...] + current: ObservedPeriodData + previous: ObservedPeriodData + last_year: ObservedPeriodData + + +class ObservedPersonPeriod(ObservedModel): + merged_prs: int + prs_per_week: float + median_merge_hours: float | None + direct_authored: int + declared_agent_owned: int + gateway_recorded_spend: float + recorded_spend_per_attributed_pr: float | None + spend_observation: Literal["records_present", "no_records"] + pr_urls: tuple[str, ...] + + +class ObservedPersonPeriods(ObservedModel): + current: ObservedPersonPeriod + previous: ObservedPersonPeriod + last_year: ObservedPersonPeriod + + +class ObservedPerson(ObservedModel): + name: str + email: str + logins: tuple[str, ...] + accounts: tuple[ObservedAccount, ...] = () + periods: ObservedPersonPeriods + + +class ObservedHumanSummary(ObservedModel): + median_merge_hours: float | None + + +class ObservedPeriod(ObservedModel): + window: ObservedWindow + merged_prs: int + median_merge_hours: float | None + human_authored: int + agent_authored: int + missing_author: int + agents_without_requester: int + matched_internal_prs: int + new_bug_labeled_issues: int | None + new_regression_labeled_issues: int | None + explicitly_titled_revert_prs: int + matched_users_recorded_spend: float + spend_observation: Literal["records_present", "no_records"] + human_summary: ObservedHumanSummary + + +class ObservedPeriods(ObservedModel): + current: ObservedPeriod + previous: ObservedPeriod + last_year: ObservedPeriod + + +class ObservedPullResponse(ObservedPull): + merge_hours: float | None + branch_cost: ROIBranchAttribution + + +class ObservedPullPeriods(ObservedModel): + current: tuple[ObservedPullResponse, ...] + previous: tuple[ObservedPullResponse, ...] + last_year: tuple[ObservedPullResponse, ...] + + +class ObservedReport(ObservedModel): + source_provider: Literal["github", "gitlab", "mixed"] + connections: tuple[ObservedSource, ...] = () + repos: tuple[str, ...] + captured_at: datetime + periods: ObservedPeriods + people: tuple[ObservedPerson, ...] + pulls: ObservedPullPeriods + unlinked_branches: tuple[ROIBranchSpend, ...] + unmatched_logins: tuple[str, ...] + + +class ObservedReportResponse(ObservedModel): + report: ObservedReport | None + + +class ObservedIdentityUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + email: str + logins: tuple[str, ...] = Field(default=(), max_length=100) + accounts: tuple[ObservedAccount, ...] | None = Field(default=None, max_length=500) + + +class ObservedConnectionIdentities(ObservedSource): + identity_map: Mapping[str, str] + unmatched_logins: tuple[str, ...] + + +class ObservedIdentities(ObservedModel): + gateway_emails: tuple[str, ...] + identity_map: Mapping[str, str] + unmatched_logins: tuple[str, ...] + connections: tuple[ObservedConnectionIdentities, ...] = () + + +class ObservedConnection(ObservedModel): + id: str = "" + source_provider: Literal["github", "gitlab"] + api_url: str + repos: tuple[str, ...] + has_token: bool + update_interval_minutes: float + ready: bool + connection_type: Literal["token", "app"] + + +class ObservedSettings(ObservedConnection): + connections: tuple[ObservedConnection, ...] = () + + +class ObservedSettingsUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + connection_id: str | None = Field(default=None, max_length=100) + source_provider: Literal["github", "gitlab"] + api_url: str + token: str | None = None + repos: tuple[str, ...] + update_interval_minutes: float | None = Field(default=None, ge=0, le=43200, allow_inf_nan=False) + + +class ObservedApp(ObservedModel): + configured: bool + can_install: bool = False + api_url: str | None = None + callback_url: str | None = None + + +class ObservedApps(ObservedModel): + github: ObservedApp + gitlab: ObservedApp + + +class ObservedAuthorization(ObservedModel): + url: str diff --git a/tests/integration/README.md b/tests/integration/README.md index e4cb3d96d59..ef418c50759 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,6 +2,8 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls +The ROI database contracts in `database/test_roi_observed.py` run in the GitHub Actions `roi-database` Postgres shard and upload coverage on each PR. They own temporary databases and script only the external provider transport. `GITHUB_FILES` in `run.py` assigns these files to GitHub Actions and excludes them from the CircleCI selection + The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate @@ -12,7 +14,7 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. Existing GitHub Actions jobs do not own these tests +The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. GitHub Actions runs only the explicit `GITHUB_FILES` set in `run.py` There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1b39eb81b01..14e31441a95 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,6 +16,7 @@ from tests.integration._support.client import Gateway, eventually, gateway_from_ from tests.integration._support.generation import LIFECYCLE_SETTINGS from tests.integration._support.manifest import OWNED_DIRECTORIES from tests.integration._support.routing import RoutingPlugin +from tests.integration.run import GITHUB_FILES COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() @@ -69,7 +70,10 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item for item in items if item.path.is_relative_to(root) and item.path.relative_to(root).parts[0] in OWNED_DIRECTORIES ) - if owned and os.environ.get("GITHUB_ACTIONS") == "true": + circleci_only: Final = tuple( + item for item in owned if item.path.relative_to(root.parents[1]).as_posix() not in GITHUB_FILES + ) + if circleci_only and os.environ.get("GITHUB_ACTIONS") == "true": raise pytest.UsageError("Integration contracts are owned by CircleCI") for item in owned: item.add_marker(pytest.mark.integration) diff --git a/tests/integration/database/test_roi_observed.py b/tests/integration/database/test_roi_observed.py new file mode 100644 index 00000000000..f64b09e85c6 --- /dev/null +++ b/tests/integration/database/test_roi_observed.py @@ -0,0 +1,823 @@ +import asyncio +from collections.abc import AsyncIterator +from datetime import date, datetime, timezone +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import httpx +import pytest +import pytest_asyncio +from fastapi import FastAPI, HTTPException +from pydantic import SecretStr + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.oauth import ( + OAuthConfig, + TokenGrant, + begin_authorization, + connected_settings, + consume_state, + exchange_code, + save_grant, +) +from litellm.proxy.roi_calculator.observed_sync import ObservedSyncManager, Progress +from litellm.proxy.roi_calculator.settings import load_settings, load_stored_settings, save_settings +from litellm.proxy.roi_calculator.sync_store import SyncStore +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.config_repository import ConfigRepository +from litellm.types.roi_observed import ObservedData, ObservedPeriodData, ObservedWindow +from tests.integration._support.database import scratch_database, write_rows + + +@pytest_asyncio.fixture(loop_scope="function") +async def repository(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[ConfigRepository]: + with scratch_database() as url: + write_rows( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB NOT NULL, ' + "last_run_at TIMESTAMP NOT NULL DEFAULT NOW(), reload_revision BIGINT NOT NULL DEFAULT 0)", + (), + database_url=url, + ) + monkeypatch.setenv("DATABASE_URL", url) + monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) + monkeypatch.setenv("LITELLM_SALT_KEY", "test-only-observed-roi-salt-0123456789") + client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + yield ConfigRepository(client, use_writer=True) + finally: + await client.disconnect() + + +def _config(provider: str = "github") -> OAuthConfig: + from pydantic import TypeAdapter + + from litellm.proxy.roi_calculator.oauth import Provider + + selected: Final = TypeAdapter(Provider).validate_python(provider) + return OAuthConfig( + selected, + "https://api.github.com" if provider == "github" else "https://gitlab.com/api/v4", + f"https://{provider}.com", + "test-client", + SecretStr("test-client-secret"), + "https://gateway.example.test", + ) + + +def _report() -> ObservedData: + period: Final = ObservedPeriodData( + window=ObservedWindow(start=date(2026, 9, 1), end=date(2026, 9, 28)), pulls=(), issues=(), spend=() + ) + return ObservedData( + source_provider="github", + source_api_url="https://api.github.com", + repos=("org/repo",), + captured_at=datetime.now(timezone.utc), + gateway_emails=(), + current=period, + previous=period, + last_year=period, + ) + + +@pytest.mark.asyncio +async def test_settings_compare_and_swap_rejects_stale_writers(repository: ConfigRepository) -> None: + assert await repository.set_param_if_revision("settings", {"revision": 1, "account": "ari"}, 0) + assert not await repository.set_param_if_revision("settings", {"revision": 1, "account": "bea"}, 0) + writes: Final = await asyncio.gather( + *(repository.set_param_if_revision("settings", {"revision": 2, "account": name}, 1) for name in ("bea", "cam")) + ) + assert sum(writes) == 1 + saved: Final = await repository.get_param("settings") + assert saved is not None and saved.param_value in ( + {"revision": 2, "account": "bea"}, + {"revision": 2, "account": "cam"}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("github", "gitlab")) +async def test_authorization_uses_pkce_single_use_state_and_encrypted_credentials( + repository: ConfigRepository, provider: str +) -> None: + config: Final = _config(provider) + url, nonce = await begin_authorization(repository, config) + params: Final = parse_qs(urlsplit(url).query) + assert params["code_challenge_method"] == ["S256"] + assert params["redirect_uri"] == [config.redirect_uri] + state: Final = await consume_state(repository, params["state"][0], nonce, config) + assert state.verifier.get_secret_value() != "**********" + + def respond(request: httpx.Request) -> httpx.Response: + body: Final = parse_qs(request.content.decode()) + assert body["code_verifier"] == [state.verifier.get_secret_value()] + assert body["code"] == ["test-code"] + assert body["client_secret"] == ["test-client-secret"] + assert body["redirect_uri"] == [config.redirect_uri] + return httpx.Response( + 200, json={"access_token": "test-access", "refresh_token": "test-refresh", "expires_in": 3600} + ) + + grant: Final = await exchange_code(config, state, "test-code", httpx.MockTransport(respond)) + await save_grant(repository, config, grant, revision=state.settings_revision) + stored: Final = await load_stored_settings(repository) + assert "test-access" not in stored.model_dump_json() and "test-refresh" not in stored.model_dump_json() + connected: Final = await load_settings(repository) + assert ( + connected.github_token if provider == "github" else connected.gitlab_token + ).get_secret_value() == "test-access" + assert connected.oauth_refresh_token.get_secret_value() == "test-refresh" + with pytest.raises(HTTPException, match="already used"): + await consume_state(repository, params["state"][0], nonce, config) + + +@pytest.mark.asyncio +async def test_authorization_rejects_a_different_browser_and_changed_settings(repository: ConfigRepository) -> None: + config: Final = _config() + url, nonce = await begin_authorization(repository, config) + state: Final = parse_qs(urlsplit(url).query)["state"][0] + with pytest.raises(HTTPException, match="same browser"): + await consume_state(repository, state, "another-browser", config) + new_url, new_nonce = await begin_authorization(repository, config) + verified: Final = await consume_state(repository, parse_qs(urlsplit(new_url).query)["state"][0], new_nonce, config) + await save_grant(repository, config, TokenGrant(access_token=SecretStr("first"))) + with pytest.raises(HTTPException, match="changed during authorization"): + await save_grant( + repository, config, TokenGrant(access_token=SecretStr("stale")), revision=verified.settings_revision + ) + assert (await load_settings(repository)).github_token.get_secret_value() == "first" + assert nonce != new_nonce + + +@pytest.mark.asyncio +async def test_concurrent_app_refresh_rotates_once_and_preserves_account_links( + repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_ID", "test-client") + monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_SECRET", "test-client-secret") + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.test") + await save_grant( + repository, + _config(), + TokenGrant(access_token=SecretStr("old-access"), refresh_token=SecretStr("old-refresh"), expires_in=1), + ) + stored: Final = await load_stored_settings(repository) + settings: Final = (await load_settings(repository)).model_copy(update={"identity_map": {"ari": "ari@example.test"}}) + await save_settings(repository, settings, stored.github_token, stored.estimator_key, revision=stored.revision) + requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + assert parse_qs(request.content.decode())["refresh_token"] == ["old-refresh"] + return httpx.Response( + 200, json={"access_token": "new-access", "refresh_token": "new-refresh", "expires_in": 3600} + ) + + results: Final = await asyncio.gather( + *(connected_settings(repository, httpx.MockTransport(respond)) for _ in range(12)) + ) + assert requests.qsize() == 1 + assert all(result.github_token.get_secret_value() == "new-access" for result in results) + assert all(result.identity_map == {"ari": "ari@example.test"} for result in results) + assert (await load_settings(repository)).oauth_refresh_token.get_secret_value() == "new-refresh" + + +@pytest.mark.asyncio +async def test_refresh_cannot_restore_a_connection_replaced_while_the_provider_responds( + repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_ID", "test-client") + monkeypatch.setenv("LITELLM_ROI_GITHUB_CLIENT_SECRET", "test-client-secret") + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.test") + await save_grant( + repository, + _config(), + TokenGrant(access_token=SecretStr("old"), refresh_token=SecretStr("refresh"), expires_in=1), + ) + started: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def respond(request: httpx.Request) -> httpx.Response: + started.set() + await release.wait() + return httpx.Response(200, json={"access_token": "late-token", "expires_in": 3600}) + + pending: Final = asyncio.create_task(connected_settings(repository, httpx.MockTransport(respond))) + await asyncio.wait_for(started.wait(), 2) + await save_grant(repository, _config(), TokenGrant(access_token=SecretStr("replacement"), expires_in=3600)) + release.set() + result: Final = await asyncio.wait_for(pending, 2) + assert result.github_token.get_secret_value() == "replacement" + assert (await load_settings(repository)).github_token.get_secret_value() == "replacement" + + +@pytest.mark.asyncio +async def test_failed_cancelled_and_stale_workers_cannot_replace_the_published_report( + repository: ConfigRepository, +) -> None: + store: Final = SyncStore(repository.prisma_client, "roi_observed") + manager: Final = ObservedSyncManager() + report: Final = _report() + await repository.set_param("roi_observed_report", report.model_dump(mode="json")) + release: Final = asyncio.Event() + + async def build(progress: Progress) -> ObservedData: + await release.wait() + raise SourceError("source failure") + + assert await manager.start(build, store) + assert not await ObservedSyncManager().start(build, store) + await store.cancel() + release.set() + await manager.cancel() + published: Final = await repository.get_param("roi_observed_report") + assert published is not None and ObservedData.model_validate(published.param_value) == report + + async def complete(progress: Progress) -> ObservedData: + return report + + assert await manager.start(complete, store) + + async def finished() -> None: + for _ in range(200): + status: Final = await store.status() + if status is not None and not status.running: + assert status.phase == "complete", status + return + await asyncio.sleep(0.01) + pytest.fail("Observed report did not publish") + + await asyncio.wait_for(finished(), 5) + saved: Final = await repository.get_param("roi_observed_report") + assert saved is not None and ObservedData.model_validate(saved.param_value) == report + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider,install_first", (("github", False), ("github", True), ("gitlab", False))) +async def test_app_callback_round_trip_and_admin_authorization( + repository: ConfigRepository, monkeypatch: pytest.MonkeyPatch, provider: str, install_first: bool +) -> None: + from fastapi import FastAPI + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.roi_observed_endpoints import ( + get_oauth_repository, + get_observed_transport, + router, + ) + from litellm.proxy.roi_calculator.settings import get_roi_config_repository + + config: Final = _config(provider) + monkeypatch.setenv(f"LITELLM_ROI_{provider.upper()}_CLIENT_ID", config.client_id) + monkeypatch.setenv(f"LITELLM_ROI_{provider.upper()}_CLIENT_SECRET", config.client_secret.get_secret_value()) + monkeypatch.setenv("PROXY_BASE_URL", config.proxy_url) + monkeypatch.setenv("LITELLM_ROI_GITHUB_APP_SLUG", "example-roi") + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "POST": + assert str(request.url) == config.token_url + if parse_qs(request.content.decode()).get("code") == ["rejected-code"]: + return httpx.Response(400, json={"error": "invalid_grant"}) + return httpx.Response( + 200, + json={ + "access_token": "provider-test-access", + "refresh_token": "provider-test-refresh", + "expires_in": 3600, + }, + ) + assert request.headers["Authorization"] == "Bearer provider-test-access" + assert "PRIVATE-TOKEN" not in request.headers + return httpx.Response(200, json=[]) + + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[get_oauth_repository] = lambda: repository + app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=config.proxy_url) as client: + for role, expected in ( + (LitellmUserRoles.INTERNAL_USER, 403), + (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, 403), + (LitellmUserRoles.PROXY_ADMIN, 200), + ): + app.dependency_overrides[user_api_key_auth] = lambda role=role: UserAPIKeyAuth(user_role=role) + response: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start") + assert response.status_code == expected, response.text + successful: Final = await client.post( + f"/roi-calculator/observed/oauth/{provider}/start", params={"install": str(install_first).lower()} + ) + assert "HttpOnly" in successful.headers["set-cookie"] and "Secure" in successful.headers["set-cookie"] + initial_state: Final = parse_qs(urlsplit(successful.json()["url"]).query)["state"][0] + installed: Final = ( + await client.get("/roi-calculator/observed/oauth/github/installed", params={"state": initial_state}) + if install_first + else None + ) + if install_first: + assert urlsplit(successful.json()["url"]).path == "/apps/example-roi/installations/new" + assert installed is not None + assert installed.status_code == 303, installed.text + assert urlsplit(installed.headers["location"]).path == "/login/oauth/authorize" + assert parse_qs(urlsplit(installed.headers["location"]).query)["state"][0] != initial_state + state: Final = ( + parse_qs(urlsplit(installed.headers["location"]).query)["state"][0] if installed else initial_state + ) + callback: Final = await client.get( + f"/roi-calculator/observed/oauth/{provider}/callback", params={"state": state, "code": "test-code"} + ) + assert callback.status_code == 303, callback.text + assert callback.headers["location"] == config.proxy_url + f"/ui/roi-calculator/?connected={provider}" + saved: Final = await client.get("/roi-calculator/observed/settings") + assert saved.json()["has_token"] is True and saved.json()["connection_type"] == "app" + assert "provider-test-access" not in saved.text and "provider-test-refresh" not in saved.text + replay: Final = await client.get( + f"/roi-calculator/observed/oauth/{provider}/callback", params={"state": state, "code": "test-code"} + ) + assert replay.status_code == 303 + assert replay.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_failed=1" + restart: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start") + denied_state: Final = parse_qs(urlsplit(restart.json()["url"]).query)["state"][0] + denied: Final = await client.get( + f"/roi-calculator/observed/oauth/{provider}/callback", + params={"state": denied_state, "error": "access_denied"}, + ) + assert denied.status_code == 303 + assert denied.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_cancelled=1" + unchanged: Final = await client.get("/roi-calculator/observed/settings") + assert unchanged.json() == saved.json() + expired_installation: Final = ( + await client.get("/roi-calculator/observed/oauth/github/installed", params={"state": initial_state}) + if install_first + else None + ) + if expired_installation is not None: + assert expired_installation.status_code == 303 + assert expired_installation.headers["location"].endswith("?connection_failed=1") + retry: Final = await client.post(f"/roi-calculator/observed/oauth/{provider}/start") + rejected: Final = await client.get( + f"/roi-calculator/observed/oauth/{provider}/callback", + params={"state": parse_qs(urlsplit(retry.json()["url"]).query)["state"][0], "code": "rejected-code"}, + ) + assert rejected.status_code == 303 + assert rejected.headers["location"] == config.proxy_url + "/ui/roi-calculator/?connection_failed=1" + assert "litellm_roi_oauth" not in client.cookies + assert (await client.get("/roi-calculator/observed/settings")).json() == saved.json() + + +@pytest.mark.asyncio +async def test_identity_api_combines_accounts_and_removes_automatic_links_without_resync( + repository: ConfigRepository, +) -> None: + from fastapi import FastAPI + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.roi_observed_endpoints import router + from litellm.proxy.roi_calculator.settings import get_roi_config_repository + from litellm.types.roi_observed import ObservedPull + + write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ()) + write_rows( + 'INSERT INTO "LiteLLM_UserTable" VALUES (%s,%s),(%s,%s)', ("ari", "ari@example.test", "bea", "bea@example.test") + ) + settings: Final = (await load_settings(repository)).model_copy(update={"repos": ("org/repo",)}) + await save_settings(repository, settings, "", "") + base: Final = _report() + pulls: Final = tuple( + ObservedPull( + repo="org/repo", + number=index, + title="Change", + url=f"https://github.com/org/repo/pull/{index}", + author=login, + profile_email="ari@example.test" if login == "ari" else "", + merged_at=base.captured_at, + ) + for index, login in enumerate(("ari", "old-ari")) + ) + data: Final = base.model_copy( + update={ + "gateway_emails": ("ari@example.test", "bea@example.test"), + "current": base.current.model_copy(update={"pulls": pulls}), + } + ) + await repository.set_param("roi_observed_report", data.model_dump(mode="json")) + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="https://gateway.example.test" + ) as client: + linked: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": ["ari", "old-ari"]} + ) + assert linked.status_code == 200, linked.text + person: Final = linked.json()["report"]["people"][0] + assert person["logins"] == ["ari", "old-ari"] and person["periods"]["current"]["merged_prs"] == 2 + conflict: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "bea@example.test", "logins": ["old-ari"]} + ) + assert conflict.status_code == 409 + unknown: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "missing@example.test", "logins": []} + ) + assert unknown.status_code == 422 + removed: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": []} + ) + assert removed.json()["report"]["people"] == [] + assert [login.split(":", 1)[-1] for login in removed.json()["report"]["unmatched_logins"]] == ["ari", "old-ari"] + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + ) + readonly: Final = await client.get("/roi-calculator/observed/report") + assert readonly.status_code == 200 + denied: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "ari@example.test", "logins": []} + ) + assert denied.status_code == 403 + + +@pytest.mark.asyncio +async def test_both_app_connections_keep_repositories_credentials_and_account_links( + repository: ConfigRepository, +) -> None: + from litellm.proxy.roi_calculator.settings import connection_id, stored_connections + + await save_grant( + repository, + _config(), + TokenGrant(access_token=SecretStr("github-access"), refresh_token=SecretStr("github-refresh")), + ) + stored: Final = await load_stored_settings(repository) + github: Final = (await load_settings(repository)).model_copy( + update={"repos": ("org/service", "org/docs"), "identity_map": {"ari": "ari@example.test"}} + ) + await save_settings(repository, github, stored.github_token, stored.estimator_key, revision=stored.revision) + await save_grant( + repository, + _config("gitlab"), + TokenGrant(access_token=SecretStr("gitlab-access"), refresh_token=SecretStr("gitlab-refresh")), + ) + gitlab_id: Final = connection_id("gitlab", _config("gitlab").api_url) + github_id: Final = connection_id("github", _config().api_url) + first: Final = await load_settings(repository, selected_id=github_id) + second: Final = await load_settings(repository, selected_id=gitlab_id) + assert first.repos == ("org/service", "org/docs") + assert first.identity_map == {"ari": "ari@example.test"} + assert first.github_token.get_secret_value() == "github-access" + assert first.oauth_refresh_token.get_secret_value() == "github-refresh" + assert second.gitlab_token.get_secret_value() == "gitlab-access" + assert second.oauth_refresh_token.get_secret_value() == "gitlab-refresh" + assert second.identity_map == {} and second.repos == () + await save_grant(repository, _config(), TokenGrant(access_token=SecretStr("github-rotated")), previous=first) + preserved: Final = await load_settings(repository, selected_id=gitlab_id) + assert preserved.model_dump(exclude={"github_token", "github_api_url"}) == second.model_dump( + exclude={"github_token", "github_api_url"} + ) + assert (await load_settings(repository, selected_id=github_id)).github_token.get_secret_value() == "github-rotated" + persisted: Final = await load_stored_settings(repository) + assert len(stored_connections(persisted)) == 2 + assert all( + token not in persisted.model_dump_json() + for token in ("github-access", "github-rotated", "gitlab-access", "github-refresh", "gitlab-refresh") + ) + + +@pytest.mark.asyncio +async def test_cross_provider_account_form_saves_atomically_without_legacy_logins(repository: ConfigRepository) -> None: + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.roi_observed_endpoints import router + from litellm.proxy.roi_calculator.settings import get_roi_config_repository, stored_connections, write_admin + + write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ()) + write_rows( + 'INSERT INTO "LiteLLM_UserTable" VALUES (%s, %s), (%s, %s)', + ("ari", "ari@example.test", "bea", "bea@example.test"), + ) + await save_grant(repository, _config("github"), TokenGrant(access_token=SecretStr("github-token"))) + await save_grant(repository, _config("gitlab"), TokenGrant(access_token=SecretStr("gitlab-token"))) + connections: Final = stored_connections(await load_stored_settings(repository)) + accounts: Final = tuple({"connection_id": entry.id, "login": "ari"} for entry in connections) + app: Final = FastAPI() + app.include_router(router) + + async def config_repository() -> ConfigRepository: + return repository + + async def admin() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + + app.dependency_overrides[get_roi_config_repository] = config_repository + app.dependency_overrides[write_admin] = admin + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client: + saved: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "ari@example.test", "accounts": accounts} + ) + assert saved.status_code == 200, saved.text + after: Final = stored_connections(await load_stored_settings(repository)) + assert all(entry.identity_map == {"ari": "ari@example.test"} for entry in after) + conflicting: Final = ({"connection_id": connections[0].id, "login": "bea"}, accounts[1]) + rejected: Final = await client.put( + "/roi-calculator/observed/identities", json={"email": "bea@example.test", "accounts": conflicting} + ) + assert rejected.status_code == 409, rejected.text + assert stored_connections(await load_stored_settings(repository)) == after + + +@pytest.mark.asyncio +async def test_http_sync_combines_providers_retains_period_and_recovers_invalid_reports( + repository: ConfigRepository, +) -> None: + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.roi_observed_endpoints import ( + get_observed_manager, + get_observed_transport, + router, + ) + from litellm.proxy.roi_calculator.settings import get_roi_config_repository + from litellm.types.roi_observed import ObservedReportResponse, ObservedSettings + + write_rows('CREATE TABLE "LiteLLM_UserTable" (user_id TEXT PRIMARY KEY, user_email TEXT)', ()) + write_rows( + 'CREATE TABLE "LiteLLM_DailyUserSpend" (user_id TEXT, date TEXT, spend DOUBLE PRECISION, api_requests INTEGER)', + (), + ) + write_rows( + 'CREATE TABLE "LiteLLM_SpendLogs" (spend DOUBLE PRECISION, request_tags JSONB, metadata JSONB, "startTime" TIMESTAMP)', + (), + ) + + def respond(request: httpx.Request) -> httpx.Response: + if request.headers.get("Authorization") == "Bearer rejected": + return httpx.Response(401, json={"message": "Bad credentials"}) + if request.url.path == "/graphql": + return httpx.Response( + 200, json={"data": {"search": {"issueCount": 0, "nodes": [], "pageInfo": {"hasNextPage": False}}}} + ) + if request.url.path.startswith("/repos/"): + return httpx.Response(200, json={"full_name": "org/service", "has_issues": True}) + if request.url.path in ("/user/repos", "/api/v4/projects"): + return httpx.Response(200, json=[]) + if request.url.path == "/api/v4/projects/org/service": + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"}) + if request.url.path.endswith(("/merge_requests", "/issues")): + return httpx.Response(200, json=[]) + raise AssertionError(str(request.url)) + + manager: Final = ObservedSyncManager() + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[get_observed_manager] = lambda: manager + app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + + async def finish() -> None: + for _ in range(200): + status: Final = await SyncStore(repository.prisma_client, "roi_observed").status() + if status and not status.running: + assert status.phase == "complete", status.error + return + await asyncio.sleep(0.01) + pytest.fail("Sync did not finish") + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client: + assert (await client.get("/roi-calculator/observed/report")).json() == {"report": None} + assert (await client.post("/roi-calculator/observed/sync")).status_code == 409 + for provider, url in (("github", "https://api.github.com"), ("gitlab", "https://gitlab.com/api/v4")): + result: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "source_provider": provider, + "api_url": url, + "token": "test-token", + "repos": ["org/service"], + "update_interval_minutes": 0, + }, + ) + assert result.status_code == 200, result.text + settings: Final = ObservedSettings.model_validate( + (await client.get("/roi-calculator/observed/settings")).json() + ) + assert len(settings.connections) == 2 and all(entry.has_token for entry in settings.connections) + for entry in settings.connections: + repositories: Final = await client.get( + "/roi-calculator/observed/repositories", params={"connection": entry.id} + ) + assert repositories.status_code == 200, repositories.text + rejected: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "source_provider": "github", + "api_url": "https://api.github.com", + "token": "rejected", + "repos": ["org/service"], + }, + ) + assert rejected.status_code == 502, rejected.text + assert ( + ObservedSettings.model_validate((await client.get("/roi-calculator/observed/settings")).json()) == settings + ) + for params in ({"days": 7}, {}): + started: Final = await client.post("/roi-calculator/observed/sync", params=params) + assert started.status_code == 202, started.text + await asyncio.wait_for(finish(), 5) + response: Final = await client.get("/roi-calculator/observed/report") + report: Final = ObservedReportResponse.model_validate(response.json()).report + assert report is not None and report.source_provider == "mixed" + assert len(report.connections) == 2 and report.periods.current.merged_prs == 0 + assert all( + (period.window.end - period.window.start).days == 6 + for period in (report.periods.current, report.periods.previous, report.periods.last_year) + ) + await repository.set_param("roi_observed_report", {"invalid": True}) + assert (await client.get("/roi-calculator/observed/report")).status_code == 500 + recovered: Final = await client.post("/roi-calculator/observed/sync") + assert recovered.status_code == 202, recovered.text + await asyncio.wait_for(finish(), 5) + rebuilt: Final = ObservedReportResponse.model_validate( + (await client.get("/roi-calculator/observed/report")).json() + ).report + assert ( + rebuilt is not None + and (rebuilt.periods.current.window.end - rebuilt.periods.current.window.start).days == 27 + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("github", "gitlab")) +async def test_host_whitespace_preserves_app_credentials_and_account_matches( + repository: ConfigRepository, provider: str +) -> None: + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.roi_observed_endpoints import get_observed_transport, router + from litellm.proxy.roi_calculator.settings import connection_id, get_roi_config_repository + + config: Final = _config(provider) + granted: Final = await save_grant( + repository, + config, + TokenGrant(access_token=SecretStr("test-access"), refresh_token=SecretStr("test-refresh"), expires_in=3600), + ) + stored: Final = await load_stored_settings(repository) + before: Final = granted.model_copy( + update={"repos": ("org/service",), "identity_map": {"ari": "ari@example.test"}, "ignored_logins": ("bea",)} + ) + await save_settings( + repository, before, stored.github_token, stored.estimator_key, stored.gitlab_token, revision=stored.revision + ) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers.get("Authorization") == "Bearer test-access" + if "/repos/" in request.url.path: + return httpx.Response(200, json={"full_name": "org/service"}) + if request.url.path.endswith("/merge_requests"): + return httpx.Response(200, json=[]) + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"}) + + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client: + response: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "connection_id": connection_id(provider, config.api_url), + "source_provider": provider, + "api_url": f" {config.api_url}/ ", + "repos": ["org/service"], + }, + ) + assert response.status_code == 200, response.text + assert (await load_settings(repository)) == before + + +@pytest.mark.asyncio +async def test_connection_edits_replace_only_the_selected_host_and_keep_the_workspace_schedule( + repository: ConfigRepository, +) -> None: + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.management_endpoints.roi_observed_endpoints import get_observed_transport, router + from litellm.proxy.roi_calculator.settings import get_roi_config_repository, stored_connections + from litellm.types.roi_calculator import ROISettings + from litellm.types.roi_observed import ObservedSettings + + legacy: Final = ROISettings(repos=("org/service",), estimator_model="legacy-model", update_interval_minutes=60) + await save_settings(repository, legacy, "", "") + + def respond(request: httpx.Request) -> httpx.Response: + if "/repos/" in request.url.path: + return httpx.Response(200, json={"full_name": "org/service"}) + if "/projects/" in request.url.path: + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/service"}) + raise AssertionError(str(request.url)) + + app: Final = FastAPI() + app.include_router(router) + app.dependency_overrides[get_roi_config_repository] = lambda: repository + app.dependency_overrides[get_observed_transport] = lambda: httpx.MockTransport(respond) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway.test") as client: + assert (await client.get("/roi-calculator/observed/settings")).status_code == 200 + assert (await load_settings(repository)) == legacy + github: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "source_provider": "github", + "api_url": "https://api.github.com", + "token": "github-token", + "repos": ["org/service"], + "update_interval_minutes": 30, + }, + ) + assert github.status_code == 200, github.text + gitlab: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "source_provider": "gitlab", + "api_url": "https://gitlab.com/api/v4", + "token": "gitlab-token", + "repos": ["org/service"], + }, + ) + assert gitlab.status_code == 200, gitlab.text + before: Final = stored_connections(await load_stored_settings(repository)) + edited: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "connection_id": github.json()["id"], + "source_provider": "github", + "api_url": "https://git.example.test/api/v3", + "token": "enterprise-token", + "repos": ["org/service"], + }, + ) + assert edited.status_code == 200, edited.text + settings: Final = ObservedSettings.model_validate(edited.json()) + assert len(settings.connections) == 2 + assert {entry.api_url for entry in settings.connections} == { + "https://git.example.test/api/v3", + "https://gitlab.com/api/v4", + } + assert all(entry.update_interval_minutes == 30 for entry in settings.connections) + after: Final = stored_connections(await load_stored_settings(repository)) + assert next(entry for entry in after if entry.source_provider == "gitlab") == next( + entry for entry in before if entry.source_provider == "gitlab" + ) + assert (await load_settings(repository)).report_mode == "observed" + stale: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "connection_id": github.json()["id"], + "source_provider": "github", + "api_url": "https://api.github.com", + "repos": [], + }, + ) + assert stale.status_code == 404, stale.text + duplicate: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "connection_id": settings.id, + "source_provider": "gitlab", + "api_url": "https://gitlab.com/api/v4", + "repos": [], + }, + ) + assert duplicate.status_code == 409, duplicate.text + assert stored_connections(await load_stored_settings(repository)) == after + manual: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "source_provider": "gitlab", + "api_url": "https://gitlab.com/api/v4", + "repos": ["org/service"], + "update_interval_minutes": 0, + }, + ) + assert manual.status_code == 200, manual.text + reselected: Final = await client.put( + "/roi-calculator/observed/settings", + json={ + "connection_id": settings.id, + "source_provider": "github", + "api_url": "https://git.example.test/api/v3", + "repos": ["org/service"], + }, + ) + assert reselected.status_code == 200, reselected.text + assert all( + entry.update_interval_minutes == 0 + for entry in ObservedSettings.model_validate(reselected.json()).connections + ) diff --git a/tests/integration/run.py b/tests/integration/run.py index c5facec0bd2..aca21ec522c 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -23,6 +23,7 @@ GROUPS: Final = MappingProxyType( "security": ("security",), } ) +GITHUB_FILES: Final = frozenset({"tests/integration/database/test_roi_observed.py"}) @dataclass(frozen=True, slots=True) @@ -63,6 +64,7 @@ def main() -> int: str(path.relative_to(root)) for folder in GROUPS[options.group] for path in sorted((root / "tests/integration" / folder).rglob("test_*.py")) + if str(path.relative_to(root)) not in GITHUB_FILES ) if options.list: print("\n".join(group_files)) diff --git a/tests/integration/spend/test_roi_branch_spend.py b/tests/integration/spend/test_roi_branch_spend.py index c90aa0073cd..9efb22c43be 100644 --- a/tests/integration/spend/test_roi_branch_spend.py +++ b/tests/integration/spend/test_roi_branch_spend.py @@ -1,6 +1,7 @@ import json import os import uuid +from collections.abc import Mapping from datetime import date from typing import Final from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit @@ -11,6 +12,8 @@ from prisma import Prisma from psycopg import sql from litellm.proxy.roi_calculator.branch_spend import read_branch_spend +from litellm.types.roi_calculator import ROIBranchSpend +from tests.integration._support.client import Gateway, JsonValue, object_value @pytest.mark.asyncio @@ -46,7 +49,7 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() -> setup.execute( sql.SQL( 'INSERT INTO {}."LiteLLM_SpendLogs" ("startTime", spend, request_tags) ' - 'VALUES (%s::timestamp, %s, %s::jsonb)' + "VALUES (%s::timestamp, %s, %s::jsonb)" ).format(sql.Identifier(schema)), (timestamp, spend, json.dumps(request_tags)), ) @@ -57,9 +60,9 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() -> (None, 100, ("litellm-roi-estimator",)), ): setup.execute( - sql.SQL('INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)').format( - sql.Identifier(schema) - ), + sql.SQL( + 'INSERT INTO {}."LiteLLM_SpendLogs" VALUES (%s::timestamp, %s, %s::jsonb, %s::jsonb)' + ).format(sql.Identifier(schema)), ( "2026-09-15 00:00:00", spend, @@ -77,3 +80,82 @@ async def test_branch_spend_uses_request_tags_once_and_respects_utc_window() -> assert costs == {"feature/one": (18, 3), "Feature/one": (7, 1), "free": (0, 1)} finally: setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +def test_documented_header_and_body_tags_reach_recorded_branch_and_pr_cost(gateway: Gateway) -> None: + import asyncio + from datetime import datetime, timezone + + from litellm.proxy.roi_calculator.branch_spend import attribute_branch_keys + from tests.integration._support.client import eventually + from tests.integration._support.database import read_rows + from tests.integration._support.wire import Reply, Request, wire_server + + marker: Final = uuid.uuid4().hex + repo: Final = f"github.com/integration/{marker}" + branch: Final = "feature/tag-attribution" + tags: Final = [f"repo:{repo}", f"branch:{branch}"] + + def respond(request: Request) -> Reply: + body: Final = object_value(json.loads(request.body)) + assert "tags" not in body and "x-litellm-tags" not in request.headers + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "owned-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + } + ).encode() + ) + + with wire_server(respond) as upstream, gateway.scenario() as scenario: + model: Final = scenario.model( + api_base=upstream.url + "/v1", input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + examples: Final[tuple[tuple[Mapping[str, JsonValue], Mapping[str, str]], ...]] = ( + ({"metadata": {"tags": tags}}, {}), + ({"tags": tags}, {}), + ({}, {"x-litellm-tags": ", ".join(tags + tags)}), + ) + for payload, headers in examples: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "tag attribution"}], + **payload, + }, + headers=headers, + ) + assert response.status_code == 200, response.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_tags @> %s::jsonb', (json.dumps(tags),) + ), + lambda values: len(values) == 3, + seconds=70, + ) + expected: Final = 3 * (5 * 0.001 + 3 * 0.002) + assert sum(float(row["spend"]) for row in rows) == pytest.approx(expected) + + async def recorded() -> tuple[ROIBranchSpend, ...]: + database: Final = Prisma() + await database.connect() + try: + today: Final = datetime.now(timezone.utc).date() + return await read_branch_spend(database, today, today, (repo,), casefold_repo=True) + finally: + await database.disconnect() + + spending: Final = asyncio.run(recorded()) + costs: Final = attribute_branch_keys(((repo, 1, repo, branch),), spending) + assert costs[(repo, 1)].spend == pytest.approx(expected) + assert costs[(repo, 1)].requests == 3 + assert costs[(repo, 1)].status == "matched" diff --git a/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py b/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py index 01d40f94d48..9c82dbac3f3 100644 --- a/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py @@ -120,6 +120,15 @@ class _ConfigRepository: self.values = MappingProxyType({**self.values, param_name: param_value}) return self.values[param_name] + async def set_param_if_revision(self, param_name: str, param_value: object, revision: int) -> bool: + from litellm.proxy.roi_calculator.settings import StoredROISettings + + stored: Final = StoredROISettings.model_validate(self.values.get(param_name, {})) + if stored.revision != revision: + return False + await self.set_param(param_name, param_value) + return True + def _client( role: LitellmUserRoles, repository: _ConfigRepository, transport: httpx.AsyncBaseTransport | None = None @@ -322,8 +331,14 @@ def test_schedule_rejects_intervals_under_five_minutes(interval: float) -> None: @pytest.mark.parametrize("anchor", ("2026-09-30T12:00:00", "2026-09-30T12:00:00Z", "2026-09-30T14:00:00+02:00")) -def test_schedule_normalizes_legacy_and_offset_timestamps(anchor: str) -> None: - settings: Final = ROISettings(repos=("example/repo",), estimator_model="estimator", update_interval_minutes=60) +@pytest.mark.parametrize("observed", (False, True)) +def test_schedule_normalizes_timestamps_and_respects_report_mode(anchor: str, observed: bool) -> None: + settings: Final = ROISettings( + repos=("example/repo",), + estimator_model="estimator", + update_interval_minutes=60, + report_mode="observed" if observed else "legacy", + ) status: Final = ROISyncStatus( running=False, phase="error", @@ -337,7 +352,8 @@ def test_schedule_normalizes_legacy_and_offset_timestamps(anchor: str) -> None: finished_at=anchor, ) report: Final = sample_report(datetime(2026, 9, 30, tzinfo=timezone.utc)) - assert _next_update(settings, status, report) == datetime(2026, 9, 30, 13, tzinfo=timezone.utc) + expected: Final = None if observed else datetime(2026, 9, 30, 13, tzinfo=timezone.utc) + assert _next_update(settings, status, report) == expected def test_manual_match_recalculates_saved_report_and_removal_restores_cohort() -> None: diff --git a/tests/unit/proxy/roi_calculator/test_github_observed.py b/tests/unit/proxy/roi_calculator/test_github_observed.py new file mode 100644 index 00000000000..8dfaac5d271 --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_github_observed.py @@ -0,0 +1,148 @@ +import asyncio +import re +from datetime import date, datetime, timedelta, timezone +from typing import Final +from unittest.mock import AsyncMock + +import httpx +import pytest +from pydantic import BaseModel, SecretStr + +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.github_observed import GitHubObserved +from litellm.types.roi_calculator import ROISettings + + +class _Variables(BaseModel): + q: str + after: str | None + + +class _Query(BaseModel): + variables: _Variables + + +def _node(number: int, merged: datetime) -> dict[str, object]: + return { + "number": number, + "url": f"https://github.com/org/repo/pull/{number}", + "title": "Change", + "createdAt": (merged - timedelta(seconds=16)).isoformat(), + "updatedAt": merged.isoformat(), + "mergedAt": merged.isoformat(), + "author": {"login": "ari", "__typename": "User"}, + } + + +def _page(nodes: tuple[dict[str, object], ...], count: int, cursor: str | None = None) -> httpx.Response: + return httpx.Response( + 200, + json={ + "data": { + "search": { + "issueCount": count, + "nodes": nodes, + "pageInfo": {"hasNextPage": cursor is not None, "endCursor": cursor}, + } + } + }, + ) + + +@pytest.mark.asyncio +async def test_large_history_splits_the_search_limit_without_losing_midnight_or_split_boundaries() -> None: + start: Final = datetime(2026, 9, 1, tzinfo=timezone.utc) + timestamps: Final = tuple(start + timedelta(seconds=index * 60) for index in range(1001)) + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["Authorization"] == "Bearer test-only-token" + assert request.url.path == "/graphql" + query: Final = _Query.model_validate_json(request.content).variables + bounds: Final = re.search(r"merged:([^ ]+)\.\.([^ ]+)", query.q) + assert bounds is not None + lower, upper = (datetime.fromisoformat(value.replace("Z", "+00:00")) for value in bounds.groups()) + assert query.q.count("merged:") == 1 + matching: Final = tuple( + _node(index, timestamp) for index, timestamp in enumerate(timestamps) if lower <= timestamp <= upper + ) + offset: Final = int(query.after or 0) + next_cursor: Final = str(offset + 100) if offset + 100 < len(matching) else None + return _page(matching[offset : offset + 100], len(matching), next_cursor) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + source: Final = GitHubObserved(ROISettings(github_token=SecretStr("test-only-token")), client) + pulls: Final = await source.pulls("org/repo", start.date(), start.date()) + assert tuple(pull.number for pull in pulls) == tuple(range(1001)) + assert all( + pull.created_at and (datetime.fromisoformat(pull.merged_at or "") - pull.created_at).total_seconds() == 16 + for pull in pulls + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ("count", "duplicate", "cursor", "partial")) +async def test_incomplete_source_results_fail_instead_of_publishing_understated_counts(failure: str) -> None: + node: Final = _node(1, datetime(2026, 9, 1, tzinfo=timezone.utc)) + + def respond(request: httpx.Request) -> httpx.Response: + if failure == "partial": + return httpx.Response(200, json={"data": None, "errors": [{"message": "permission denied"}]}) + if failure == "duplicate": + return _page((node, node), 2) + if failure == "cursor": + return _page((node,), 2, "repeated") + return _page((node,), 2) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + source: Final = GitHubObserved(ROISettings(), client) + with pytest.raises(SourceError): + await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1)) + + +@pytest.mark.asyncio +async def test_disabled_issue_tracking_is_unknown_instead_of_zero_bugs() -> None: + def respond(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + return httpx.Response(200, json={"has_issues": False}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + source: Final = GitHubObserved(ROISettings(), client) + assert await source.issues("org/repo", date(2026, 9, 1), date(2026, 9, 1)) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ("timeout", "unavailable", "rate_limit")) +async def test_read_queries_recover_from_temporary_provider_failures( + failure: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + responses: Final = iter((False, False, True)) + node: Final = _node(1, datetime(2026, 9, 1, tzinfo=timezone.utc)) + + def respond(request: httpx.Request) -> httpx.Response: + if next(responses): + return _page((node,), 1) + if failure == "timeout": + raise httpx.ReadTimeout("scripted timeout", request=request) + return httpx.Response(429 if failure == "rate_limit" else 502) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + source: Final = GitHubObserved(ROISettings(), client) + pulls: Final = await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1)) + assert tuple(pull.number for pull in pulls) == (1,) + + +@pytest.mark.asyncio +async def test_read_retries_stop_after_three_attempts(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + return httpx.Response(503) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client: + source: Final = GitHubObserved(ROISettings(), client) + with pytest.raises(SourceError, match="HTTP 503"): + await source.pulls("org/repo", date(2026, 9, 1), date(2026, 9, 1)) + assert requests.qsize() == 3 diff --git a/tests/unit/proxy/roi_calculator/test_gitlab.py b/tests/unit/proxy/roi_calculator/test_gitlab.py index 260a1bd9b7e..e3b237a1d41 100644 --- a/tests/unit/proxy/roi_calculator/test_gitlab.py +++ b/tests/unit/proxy/roi_calculator/test_gitlab.py @@ -279,3 +279,28 @@ async def test_gitlab_retries_transient_errors_and_checks_merge_request_access() assert next(statuses, None) is None finally: await source.close() + + +@pytest.mark.asyncio +async def test_observed_issues_preserve_last_second_boundaries_and_disabled_tracking() -> None: + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/projects/org/disabled"): + return httpx.Response(200, json={"id": 2, "path_with_namespace": "org/disabled", "issues_enabled": False}) + if request.url.path.endswith("/projects/org/repo"): + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"}) + assert request.url.params["created_before"] == "2026-10-01T00:00:00Z" + return httpx.Response( + 200, + json=[ + {"iid": 1, "created_at": "2026-09-30T23:59:59.999Z", "labels": ["type::bug"]}, + {"iid": 2, "created_at": "2026-10-01T00:00:00Z", "labels": ["bug"]}, + ], + ) + + source: Final = GitLab(ROISettings(source_provider="gitlab"), httpx.MockTransport(respond)) + try: + issues: Final = await source.issues("org/repo", date(2026, 9, 1), date(2026, 9, 30)) + assert issues is not None and tuple(issue.number for issue in issues) == (1,) + assert await source.issues("org/disabled", date(2026, 9, 1), date(2026, 9, 30)) is None + finally: + await source.close() diff --git a/tests/unit/proxy/roi_calculator/test_oauth.py b/tests/unit/proxy/roi_calculator/test_oauth.py new file mode 100644 index 00000000000..a8b8716fc4c --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_oauth.py @@ -0,0 +1,25 @@ +from dataclasses import replace +from typing import Final + +from pydantic import SecretStr + +from litellm.proxy.roi_calculator.oauth import OAuthConfig + + +def test_app_urls_support_enterprise_and_a_gateway_path_prefix() -> None: + cloud: Final = OAuthConfig( + "github", + "https://api.github.com", + "https://github.com", + "test-client", + SecretStr("test-secret"), + "https://gateway.example.test/proxy", + "test-app", + ) + enterprise: Final = replace(cloud, api_url="https://git.example.test/api/v3", base_url="https://git.example.test") + assert cloud.installation_url == cloud.base_url + "/apps/test-app/installations/new" + assert enterprise.installation_url == enterprise.base_url + "/github-apps/test-app/installations/new" + assert enterprise.cookie_path == "/proxy/roi-calculator/observed/oauth" + assert enterprise.redirect_uri.startswith(enterprise.proxy_url + "/roi-calculator/") + assert replace(cloud, provider="gitlab").installation_url is None + assert replace(cloud, app_slug="").installation_url is None diff --git a/tests/unit/proxy/roi_calculator/test_observed_analytics.py b/tests/unit/proxy/roi_calculator/test_observed_analytics.py new file mode 100644 index 00000000000..a8416f985da --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_observed_analytics.py @@ -0,0 +1,176 @@ +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.proxy.roi_calculator.observed_analytics import ( + declared_requester, + merge_hours, + reporting_windows, + summarize_observed, +) +from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow + +_NOW: Final = datetime(2026, 10, 3, tzinfo=timezone.utc) +_WINDOW: Final = ObservedWindow(start=date(2026, 9, 5), end=date(2026, 10, 2)) + + +def _pull(login: str, number: int = 1, repo: str = "org/service", **fields: object) -> ObservedPull: + return ObservedPull.model_validate( + { + "repo": repo, + "number": number, + "title": "Ship change", + "url": f"https://github.com/{repo}/pull/{number}", + "author": login, + "created_at": "2026-09-10T00:00:00Z", + "merged_at": "2026-09-10T00:01:19Z", + **fields, + } + ) + + +def _data(current: ObservedPeriodData, previous: ObservedPeriodData | None = None) -> ObservedData: + empty: Final = ObservedPeriodData(window=_WINDOW, pulls=(), issues=(), spend=()) + return ObservedData( + source_provider="github", + source_api_url="https://api.github.com", + repos=("org/service",), + captured_at=_NOW, + gateway_emails=("ari@example.test", "bea@example.test"), + current=current, + previous=previous or empty, + last_year=empty, + ) + + +def test_multiple_accounts_share_one_cost_denominator_and_pr_numbers_are_scoped_to_repositories() -> None: + direct: Final = _pull("ari", profile_email="ari@example.test") + alternate: Final = _pull("old-ari", repo="org/other") + agent: Final = _pull("devin-ai[bot]", 3, agent=True, requester="old-ari") + unowned: Final = _pull("devin-ai[bot]", 4, agent=True) + period: Final = ObservedPeriodData( + window=_WINDOW, + pulls=(direct, alternate, agent, unowned), + issues=(), + spend=({"email": "ari@example.test", "spend": 90.0, "date": "2026-09-10", "user_id": "ari", "requests": 1},), + ) + report: Final = summarize_observed(_data(period), {"old-ari": "ari@example.test"}) + assert len(report.people) == 1 + person: Final = report.people[0] + assert person.logins == ("ari", "old-ari") + assert person.periods.current.pr_urls == (direct.url, alternate.url, agent.url) + assert (person.periods.current.direct_authored, person.periods.current.declared_agent_owned) == (2, 1) + assert person.periods.current.recorded_spend_per_attributed_pr == 30.0 + assert person.periods.current.prs_per_week == 0.75 + assert (report.periods.current.merged_prs, report.periods.current.matched_internal_prs) == (4, 3) + assert report.periods.current.agents_without_requester == 1 + + +def test_missing_spend_stays_unknown_and_a_recorded_zero_stays_zero() -> None: + period: Final = ObservedPeriodData( + window=_WINDOW, + pulls=(_pull("ari"), _pull("bea", 2)), + issues=None, + spend=({"email": "bea@example.test", "spend": 0.0, "date": "2026-09-10", "user_id": "bea", "requests": 1},), + ) + report: Final = summarize_observed(_data(period), {"ari": "ari@example.test", "bea": "bea@example.test"}) + assert tuple(person.periods.current.recorded_spend_per_attributed_pr for person in report.people) == (None, 0.0) + assert tuple(person.periods.current.spend_observation for person in report.people) == ( + "no_records", + "records_present", + ) + assert report.periods.current.new_bug_labeled_issues is None + assert report.periods.previous.new_bug_labeled_issues == 0 + assert report.people[0].periods.previous.recorded_spend_per_attributed_pr is None + + +def test_manual_links_override_automatic_matches_and_removal_suppresses_rematching() -> None: + period: Final = ObservedPeriodData( + window=_WINDOW, pulls=(_pull("ari", profile_email="ari@example.test"),), issues=(), spend=() + ) + data: Final = _data(period) + assert summarize_observed(data, {}).people[0].email == "ari@example.test" + assert summarize_observed(data, {"ari": "bea@example.test"}).people[0].email == "bea@example.test" + removed: Final = summarize_observed(data, {}, ("ari",)) + assert removed.people == () + assert removed.unmatched_logins == ("ari",) + assert summarize_observed(data, {"ari": "bea@example.test"}, ("ari",)).people[0].email == "bea@example.test" + + +def test_conflicting_public_emails_do_not_silently_choose_an_owner() -> None: + current: Final = ObservedPeriodData( + window=_WINDOW, pulls=(_pull("ari", profile_email="ari@example.test"),), issues=(), spend=() + ) + previous: Final = current.model_copy(update={"pulls": (_pull("ari", profile_email="bea@example.test"),)}) + report: Final = summarize_observed(_data(current, previous), {}) + assert report.people == () + assert report.unmatched_logins == ("ari",) + + +def test_quality_counts_labelled_issues_once_and_does_not_infer_bugs_from_pr_titles() -> None: + period: Final = ObservedPeriodData( + window=_WINDOW, + pulls=(_pull("ari", title="fix: critical bug"), _pull("ari", 2, title='Revert "change"')), + issues=tuple( + ObservedIssue(repo="org/service", number=index, created_at=_NOW, labels=labels) + for index, labels in enumerate( + ( + ("BUG", "kind:bug"), + ("type::bug", "type::regression"), + ("debug",), + ) + ) + ), + spend=(), + ) + report: Final = summarize_observed(_data(period), {}) + assert ( + report.periods.current.new_bug_labeled_issues, + report.periods.current.new_regression_labeled_issues, + report.periods.current.explicitly_titled_revert_prs, + ) == (2, 1, 1) + + +@pytest.mark.parametrize( + "created,merged,expected", + ( + (None, "2026-09-10T00:00:16Z", None), + ("2026-09-10T00:00:00Z", "2026-09-10T00:00:16Z", 16 / 3600), + ("2026-09-10T00:00:00Z", "2026-09-10T00:00:00Z", 0), + ("2026-09-10T00:00:01Z", "2026-09-10T00:00:00Z", None), + ("2026-09-10T00:00:00", "2026-09-10T00:00:16Z", None), + ), +) +def test_merge_duration_preserves_seconds_and_rejects_invalid_intervals( + created: str | None, merged: str, expected: float | None +) -> None: + assert merge_hours(_pull("ari", created_at=created, merged_at=merged)) == expected + + +@pytest.mark.parametrize( + "now", (datetime(2024, 3, 1, tzinfo=timezone.utc), _NOW, _NOW.replace(tzinfo=timezone(timedelta(hours=14)))) +) +@pytest.mark.parametrize("days", (1, 7, 28, 90, 366)) +def test_reporting_windows_have_equal_lengths_and_exclude_today(now: datetime, days: int) -> None: + current, previous, yearly = reporting_windows(now, days) + assert all((window.end - window.start).days + 1 == days for window in (current, previous, yearly)) + assert current.end == now.astimezone(timezone.utc).date() - timedelta(days=1) + assert previous.end == current.start - timedelta(days=1) + assert yearly.end.year == current.end.year - 1 + assert yearly.end.month == current.end.month + + +@pytest.mark.parametrize( + "author,body,expected", + ( + ("devin-ai[bot]", "Requested by: @Ari", "ari"), + ("devin-ai-integration", "Requested by: @Ari", "ari"), + ("devin-ai", "Requested by: @Ari", "ari"), + ("human", "Requested by: @ari", ""), + ("devin-ai[bot]", "Requested by: @ari\nRequested by: @bea", ""), + ("devin-ai[bot]", "Mentions @ari", ""), + ), +) +def test_agent_ownership_requires_one_explicit_requester(author: str, body: str, expected: str) -> None: + assert declared_requester(author, body) == expected diff --git a/tests/unit/proxy/roi_calculator/test_observed_sync.py b/tests/unit/proxy/roi_calculator/test_observed_sync.py new file mode 100644 index 00000000000..55ffe401286 --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_observed_sync.py @@ -0,0 +1,140 @@ +from datetime import date, datetime, timezone +from typing import Final, Literal + +import httpx +import pytest +from pydantic import BaseModel, SecretStr + +from litellm.proxy.roi_calculator.observed_analytics import summarize_observed +from litellm.proxy.roi_calculator.observed_sync import collect_observed +from litellm.proxy.roi_calculator.source import repository_tag +from litellm.types.roi_calculator import ROIBranchSpend, ROISettings, ROISpendRecord + + +class _Variables(BaseModel): + q: str + + +class _Query(BaseModel): + variables: _Variables + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("github", "gitlab")) +@pytest.mark.parametrize("days", (7, 28, 90)) +@pytest.mark.parametrize("include_disabled_repo", (False, True)) +async def test_live_provider_metadata_reaches_people_quality_durations_and_branch_spend_without_an_estimator( + provider: Literal["github", "gitlab"], + days: int, + include_disabled_repo: bool, +) -> None: + settings: Final = ROISettings( + source_provider=provider, + repos=("org/repo", "org/disabled") if include_disabled_repo else ("org/repo",), + github_token=SecretStr("source-test-token"), + gitlab_token=SecretStr("source-test-token"), + identity_map={"old-ari": "ari@example.test"}, + ) + tag: Final = repository_tag(settings, "org/repo") + + async def spend(start: date, end: date) -> tuple[ROISpendRecord, ...]: + return ({"date": str(start), "user_id": "ari", "email": "ari@example.test", "spend": 30.0, "requests": 10},) + + async def users() -> frozenset[str]: + return frozenset(("ari@example.test",)) + + async def branches(start: date, end: date, repos: tuple[str, ...]) -> tuple[ROIBranchSpend, ...]: + assert repos == tuple(sorted(repository_tag(settings, repo) for repo in settings.repos)) + return (ROIBranchSpend(repo=tag, branch="fix/parser", spend=5.0, requests=2),) + + def respond(request: httpx.Request) -> httpx.Response: + path: Final = request.url.path + if path == "/graphql": + query: Final = _Query.model_validate_json(request.content).variables.q + if "repo:org/disabled " in query: + return httpx.Response( + 200, json={"data": {"search": {"issueCount": 0, "nodes": [], "pageInfo": {"hasNextPage": False}}}} + ) + kind: Final = "pull" if "is:pr" in query else "issue" + start: Final = query.split("merged:" if kind == "pull" else "created:")[1][:10] + node: Final = { + "number": 1, + "url": "https://github.com/org/repo/pull/1", + "title": "Change", + "createdAt": f"{start}T12:00:00Z", + "updatedAt": f"{start}T12:00:16Z", + "mergedAt": f"{start}T12:00:16Z", + "author": {"login": "devin-ai-integration", "__typename": "Bot"}, + "body": "Requested by: @old-ari", + "headRefName": "fix/parser", + "headRepository": {"nameWithOwner": "org/repo"}, + "labels": {"nodes": [{"name": "bug"}], "pageInfo": {"hasNextPage": False}}, + } + return httpx.Response( + 200, json={"data": {"search": {"issueCount": 1, "nodes": [node], "pageInfo": {"hasNextPage": False}}}} + ) + if path == "/repos/org/repo": + return httpx.Response(200, json={"has_issues": True}) + if path == "/repos/org/disabled": + return httpx.Response(200, json={"has_issues": False}) + if path.endswith("/projects/org/disabled"): + return httpx.Response(200, json={"id": 2, "path_with_namespace": "org/disabled", "issues_enabled": False}) + if path.endswith("/projects/2/merge_requests"): + return httpx.Response(200, json=[]) + if path.endswith("/projects/org/repo"): + return httpx.Response(200, json={"id": 1, "path_with_namespace": "org/repo"}) + if path.endswith("/merge_requests"): + start: Final = request.url.params["merged_after"][:10] + return httpx.Response( + 200, + json=[ + { + "iid": 1, + "web_url": "https://gitlab.com/org/repo/-/merge_requests/1", + "title": "Change", + "author": {"username": "old-ari"}, + "created_at": f"{start}T12:00:00Z", + "updated_at": f"{start}T12:00:16Z", + "merged_at": f"{start}T12:00:16Z", + "source_branch": "fix/parser", + "source_project_id": 1, + } + ], + ) + if path.endswith("/issues"): + start: Final = request.url.params["created_after"][:10] + return httpx.Response(200, json=[{"iid": 1, "created_at": f"{start}T12:00:00Z", "labels": ["bug"]}]) + raise AssertionError(f"Unexpected API request: {request.method} {path}") + + data: Final = await collect_observed( + settings, + spend, + users, + branches, + datetime(2026, 10, 3, tzinfo=timezone.utc), + lambda stage, done, total: None, + httpx.MockTransport(respond), + days=days, + ) + assert all( + (period.window.end - period.window.start).days + 1 == days + for period in (data.current, data.previous, data.last_year) + ) + report: Final = summarize_observed(data, settings.identity_map) + person: Final = report.people[0].periods.current + assert (person.merged_prs, person.gateway_recorded_spend, person.recorded_spend_per_attributed_pr) == ( + 1, + 30.0, + 30.0, + ) + assert person.median_merge_hours == 16 / 3600 + assert person.declared_agent_owned == (1 if provider == "github" else 0) + assert tuple( + window.merged_prs for window in (report.periods.current, report.periods.previous, report.periods.last_year) + ) == (1, 1, 1) + assert tuple( + period.new_bug_labeled_issues + for period in (report.periods.current, report.periods.previous, report.periods.last_year) + ) == (1, 1, 1) + assert report.pulls.current[0].branch_cost.spend == 5.0 + assert report.unlinked_branches == () diff --git a/tests/unit/proxy/roi_calculator/test_observed_workspace.py b/tests/unit/proxy/roi_calculator/test_observed_workspace.py new file mode 100644 index 00000000000..42ea77caf54 --- /dev/null +++ b/tests/unit/proxy/roi_calculator/test_observed_workspace.py @@ -0,0 +1,136 @@ +from datetime import date, datetime, timezone +from typing import Final + +import pytest + +from litellm.proxy.roi_calculator.github import SourceError +from litellm.proxy.roi_calculator.observed_workspace import ( + combine_observed, + scoped_data, + source_details, + summarize_workspace, +) +from litellm.proxy.roi_calculator.settings import StoredConnection +from litellm.types.roi_calculator import ROIBranchSpend, ROISettings +from litellm.types.roi_observed import ObservedData, ObservedIssue, ObservedPeriodData, ObservedPull, ObservedWindow + + +def _source(settings: ROISettings, issues: tuple[ObservedIssue, ...] | None = ()) -> ObservedData: + host: Final = "github.com" if settings.source_provider == "github" else "gitlab.com" + period: Final = ObservedPeriodData( + window=ObservedWindow(start=date(2026, 9, 1), end=date(2026, 9, 28)), + pulls=tuple( + ObservedPull( + repo=repo, + number=1, + title="Change", + url=f"https://{host}/{repo}/pull/1", + author="ari", + created_at=datetime(2026, 9, 10, 0, 0, 0, tzinfo=timezone.utc), + merged_at=datetime(2026, 9, 10, 0, 0, 30, tzinfo=timezone.utc), + source_repo=f"{host}/{repo}", + source_branch="feature/one", + ) + for repo in settings.repos + ), + issues=issues, + spend=({"date": "2026-09-10", "user_id": "ari", "email": "ari@example.test", "spend": 60.0, "requests": 3},), + branch_spend=tuple( + ROIBranchSpend(repo=f"{host}/{repo}", branch="feature/one", spend=2, requests=1) for repo in settings.repos + ), + ) + data: Final = ObservedData( + source_provider=settings.source_provider, + source_api_url=settings.source_api_url, + repos=settings.repos, + captured_at=datetime(2026, 9, 29, tzinfo=timezone.utc), + gateway_emails=("ari@example.test", "bea@example.test"), + current=period, + previous=period, + last_year=period, + ) + return scoped_data(data, source_details(settings)) + + +@pytest.mark.parametrize("same_person", (True, False)) +def test_multiple_repos_and_providers_scope_usernames_and_count_spend_once(same_person: bool) -> None: + github: Final = ROISettings(repos=("org/service", "org/docs")) + gitlab: Final = ROISettings(source_provider="gitlab", repos=("org/service",)) + combined: Final = combine_observed( + (_source(github), _source(gitlab)), ("github.com/org/service", "github.com/org/docs", "gitlab.com/org/service") + ) + report: Final = summarize_workspace( + combined, + ( + StoredConnection( + source_provider="github", api_url=github.source_api_url, identity_map={"ari": "ari@example.test"} + ), + StoredConnection( + source_provider="gitlab", + api_url=gitlab.source_api_url, + identity_map={"ari": "ari@example.test" if same_person else "bea@example.test"}, + ), + ), + ) + person: Final = next(person for person in report.people if person.email == "ari@example.test") + assert report.source_provider == "mixed" + assert report.periods.current.merged_prs == 3 + assert report.periods.current.matched_users_recorded_spend == 60 + assert person.periods.current.merged_prs == (3 if same_person else 2) + assert person.periods.current.gateway_recorded_spend == 60 + assert person.periods.current.recorded_spend_per_attributed_pr == (20 if same_person else 30) + assert len(person.accounts) == (2 if same_person else 1) + assert all(pull.branch_cost.spend == 2 for pull in report.pulls.current) + assert report.unlinked_branches == () + + +def test_empty_repository_is_a_successful_zero_activity_report() -> None: + settings: Final = ROISettings(repos=()) + data: Final = combine_observed((_source(settings),), ("org/empty",)) + report: Final = summarize_workspace(data, ()) + assert report.periods.current.merged_prs == 0 + assert report.periods.current.new_bug_labeled_issues == 0 + assert report.periods.current.median_merge_hours is None + assert report.people == () and report.pulls.current == () + + +@pytest.mark.parametrize( + ("issues", "expected"), + ( + (None, None), + ((), 0), + ( + ( + ObservedIssue( + repo="org/service", + number=1, + created_at=datetime(2026, 9, 10, tzinfo=timezone.utc), + labels=("bug", "regression"), + ), + ), + 1, + ), + ), +) +def test_disabled_tracking_does_not_hide_other_connections_quality_counts( + issues: tuple[ObservedIssue, ...] | None, expected: int | None +) -> None: + github: Final = _source(ROISettings(repos=("org/docs",)), issues=None) + gitlab: Final = _source(ROISettings(source_provider="gitlab", repos=("org/service",)), issues=issues) + report: Final = summarize_workspace(combine_observed((github, gitlab), ("org/docs", "org/service")), ()) + assert tuple( + (period.new_bug_labeled_issues, period.new_regression_labeled_issues) + for period in (report.periods.current, report.periods.previous, report.periods.last_year) + ) == ((expected, expected),) * 3 + assert report.periods.current.merged_prs == 2 + + +def test_duplicate_connection_cannot_double_count_a_merged_change() -> None: + data: Final = _source(ROISettings(repos=("org/service",))) + with pytest.raises(SourceError, match="more than one connection"): + combine_observed((data, data), data.repos) + + +def test_github_repository_selection_deduplicates_case_variants() -> None: + settings: Final = ROISettings(repos=("Org/Service", "org/service", "org/docs", "org/docs.git")) + assert settings.repos == ("Org/Service", "org/docs") diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index cc25627c651..5f3903a5feb 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -73,6 +73,19 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: assert [(finding.subject, finding.detail) for finding in findings] == [ (test_path, "integration contract is also selected by GitHub Actions") ] + github_path: Final = "tests/integration/management/test_github_contract.py" + (tmp_path / github_path).write_text("def test_contract(): pass\n") + runner: Final = tmp_path / "tests/integration/run.py" + runner.write_text(runner.read_text() + f"GITHUB_FILES: Final = frozenset({{{github_path!r}}})\n") + workflow.write_text(yaml.safe_dump({"jobs": {"tests": {"steps": [{"run": f"pytest {github_path}"}]}}})) + github_owned, github_findings = coverage._integration_ownership(tmp_path) + assert github_owned == frozenset({test_path, github_path}) + assert github_findings == () + workflow.write_text(yaml.safe_dump({"jobs": {}})) + _, missing_invocation = coverage._integration_ownership(tmp_path) + assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ + (github_path, "GitHub-owned integration contract has no invoking workflow") + ] def test_an_ancestor_directory_covers_a_file_but_does_not_name_it(): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.integration.test.tsx new file mode 100644 index 00000000000..bc242ebe14a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.integration.test.tsx @@ -0,0 +1,88 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, expect, it, vi } from "vitest"; +import ObservedAccounts from "./ObservedAccounts"; + +afterEach(() => vi.unstubAllGlobals()); +it("links several usernames on both providers to one email in a single save", async () => { + const writes: unknown[] = []; + const saved = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string, init: RequestInit) => { + if (init.method === "PUT") { + writes.push(JSON.parse(String(init.body))); + return Response.json({ report: null }); + } + const identities = { + gateway_emails: ["ari@example.test"], + identity_map: { old: "ari@example.test" }, + unmatched_logins: ["new"], + connections: [ + { + id: "github-id", + source_provider: "github", + api_url: "https://api.github.com", + identity_map: { old: "ari@example.test" }, + unmatched_logins: ["new"], + }, + { + id: "gitlab-id", + source_provider: "gitlab", + api_url: "https://gitlab.com/api/v4", + identity_map: {}, + unmatched_logins: ["new"], + }, + ], + }; + return Response.json(identities); + }), + ); + const user = userEvent.setup(); + render(); + await screen.findByLabelText(/GitHub usernames/); + fireEvent.change(screen.getByLabelText("Internal email"), { target: { value: "ari@example.test" } }); + expect(screen.getByLabelText(/GitHub usernames/)).toHaveValue("old"); + fireEvent.change(screen.getByLabelText(/GitHub usernames/), { target: { value: "@Old, new, NEW" } }); + fireEvent.change(screen.getByLabelText(/GitLab usernames/), { target: { value: "new" } }); + await user.click(screen.getByRole("button", { name: "Save accounts" })); + await waitFor(() => expect(saved).toHaveBeenCalledOnce()); + expect(writes).toEqual([ + { + email: "ari@example.test", + accounts: [ + { connection_id: "github-id", login: "old" }, + { connection_id: "github-id", login: "new" }, + { connection_id: "gitlab-id", login: "new" }, + ], + }, + ]); +}); +it("keeps a conflicting link editable", async () => { + const saved = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string, init: RequestInit) => + init.method === "PUT" + ? Response.json({ detail: "An account is already linked to another email. Unlink it first" }, { status: 409 }) + : Response.json({ gateway_emails: ["ari@example.test"], identity_map: {}, unmatched_logins: [] }), + ), + ); + const user = userEvent.setup(); + render( + , + ); + fireEvent.change(screen.getByLabelText("Source usernames"), { target: { value: "old, new" } }); + const button = screen.getByRole("button", { name: "Save accounts" }); + await waitFor(() => expect(button).toBeEnabled()); + await user.click(button); + expect(await screen.findByRole("alert")).toHaveTextContent("An account is already linked to another email"); + expect(screen.getByLabelText("Source usernames")).toHaveValue("old, new"); + expect(saved).not.toHaveBeenCalled(); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.tsx new file mode 100644 index 00000000000..ab6c4cf38df --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedAccounts.tsx @@ -0,0 +1,209 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { z } from "zod"; +import { apiClient } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog"; +import { accountLogins, type ObservedPerson } from "./observedData"; + +const connectionIdentityFields = { + id: z.string(), + source_provider: z.enum(["github", "gitlab"]), + api_url: z.string(), + identity_map: z.record(z.string(), z.string()), + unmatched_logins: z.array(z.string()), +}; +const identitiesFields = { + gateway_emails: z.array(z.string()), + identity_map: z.record(z.string(), z.string()), + unmatched_logins: z.array(z.string()), + connections: z.array(z.object(connectionIdentityFields)).optional(), +}; +const identitiesSchema = z.object(identitiesFields); + +function matches(identities: z.infer, email: string, people: ObservedPerson[]) { + const person = people.find((entry) => entry.email === email); + return Object.fromEntries( + (identities.connections ?? []).map((entry) => [ + entry.id, + [ + ...new Set([ + ...Object.entries(entry.identity_map) + .filter(([, address]) => address === email) + .map(([login]) => login), + ...(person?.accounts ?? []) + .filter((account) => account.connection_id === entry.id) + .map((account) => account.login), + ]), + ].join(", "), + ]), + ); +} + +export default function ObservedAccounts({ + accessToken, + people, + initialEmail = "", + onClose, + onSaved, +}: { + accessToken: string; + people: ObservedPerson[]; + initialEmail?: string; + onClose: () => void; + onSaved: () => void; +}) { + const [identities, setIdentities] = useState | null>(null); + const [email, setEmail] = useState(initialEmail); + const [logins, setLogins] = useState(people.find((person) => person.email === initialEmail)?.logins.join(", ") ?? ""); + const [linked, setLinked] = useState>({}); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(""); + useEffect(() => { + const controller = new AbortController(); + apiClient + .get("/roi-calculator/observed/identities", { accessToken, signal: controller.signal }) + .then((data) => { + if (!controller.signal.aborted) { + const parsed = identitiesSchema.parse(data); + setIdentities(parsed); + setLinked(matches(parsed, initialEmail, people)); + } + }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason)); + }); + return () => controller.abort(); + }, [accessToken, initialEmail, people]); + function selectEmail(value: string) { + setEmail(value); + if (identities) setLinked(matches(identities, value, people)); + const automatic = people.find((person) => person.email === value)?.logins ?? []; + const manual = Object.entries(identities?.identity_map ?? {}) + .filter(([, address]) => address === value) + .map(([login]) => login); + setLogins([...new Set([...automatic, ...manual])].join(", ")); + } + async function save() { + setSaving(true); + setError(""); + try { + await apiClient.put("/roi-calculator/observed/identities", { + accessToken, + body: { + email: email.trim().toLowerCase(), + ...(identities?.connections?.length + ? { + accounts: identities.connections.flatMap((entry) => + accountLogins(linked[entry.id] ?? "").map((login) => ({ connection_id: entry.id, login })), + ), + } + : { logins: accountLogins(logins) }), + }, + }); + onSaved(); + onClose(); + } catch (reason) { + setError(extractProxyErrorMessage(reason)); + } finally { + setSaving(false); + } + } + return ( + { + if (!open) onClose(); + }} + > + + + Link accounts + Match one internal user to all their source accounts + +
+
+ + selectEmail(event.target.value)} + /> + + {identities?.gateway_emails.map((address) => +
+ {identities?.connections?.length ? ( + identities.connections.map((entry) => ( +
+ + setLinked({ ...linked, [entry.id]: event.target.value })} + placeholder="current-account, old-account" + /> + {entry.unmatched_logins.length > 0 && ( +
+ {entry.unmatched_logins.length} unmatched accounts +
+ {entry.unmatched_logins.map((login) => ( + + ))} +
+
+ )} +
+ )) + ) : ( +
+ + setLogins(event.target.value)} + placeholder="current-account, old-account" + /> +
+ )} +

+ Separate accounts with commas. Their merged changes are combined, and gateway spend is counted once +

+ {error && ( +

+ {error} +

+ )} + +
+ + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.integration.test.tsx new file mode 100644 index 00000000000..730f0c8faf5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.integration.test.tsx @@ -0,0 +1,198 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import ObservedConnections from "./ObservedConnections"; +import type { ObservedSettings } from "./observedData"; + +const settings: ObservedSettings = { + id: "initial-github", + source_provider: "github", + api_url: "https://api.github.com", + repos: [], + has_token: false, + connection_type: "token", + update_interval_minutes: 1440, + ready: false, +}; +const app = { configured: true, api_url: null, callback_url: null }; +afterEach(() => vi.unstubAllGlobals()); + +describe("observed ROI connections", () => { + it("identifies the saved connection when editing its host", async () => { + const writes: unknown[] = []; + const existing = { ...settings, id: "saved-github", has_token: true, repos: ["org/service"], ready: true }; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app }); + if (path.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false }); + if (path.endsWith("/settings")) { + writes.push(JSON.parse(String(init.body))); + return Response.json({ ...existing, id: "enterprise-github", api_url: "https://git.example.test/api/v3" }); + } + throw new Error(path); + }), + ); + const user = userEvent.setup(); + render( + , + ); + await user.click(screen.getByRole("button", { name: "Edit GitHub api.github.com" })); + await user.click(screen.getByRole("button", { name: "Change connection" })); + await user.click(screen.getByText("Self-hosted instance")); + fireEvent.change(screen.getByLabelText("API URL"), { target: { value: "https://git.example.test/api/v3" } }); + fireEvent.change(screen.getByLabelText("GitHub access token"), { target: { value: "enterprise-test-token" } }); + await user.click(screen.getByRole("button", { name: "Continue" })); + expect(await screen.findByRole("heading", { name: "Choose repositories" })).toBeInTheDocument(); + expect(writes).toEqual([ + { + connection_id: "saved-github", + source_provider: "github", + api_url: "https://git.example.test/api/v3", + token: "enterprise-test-token", + repos: [], + update_interval_minutes: 1440, + }, + ]); + }); + it("starts a GitHub installation when switching from a GitLab app connection", async () => { + const starts = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const url = new URL(input, "http://localhost"); + if (url.pathname.endsWith("/apps")) + return Response.json({ github: { ...app, can_install: true }, gitlab: app }); + if (url.pathname.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false }); + starts(url.searchParams.get("install"), init.credentials); + return Response.json({ detail: "Authorization test stopped before redirect" }, { status: 502 }); + }), + ); + const user = userEvent.setup(); + const connected: ObservedSettings = { + ...settings, + source_provider: "gitlab", + connection_type: "app", + has_token: true, + }; + render(); + await user.click(screen.getByRole("button", { name: "Change connection" })); + await user.click(screen.getByRole("button", { name: "GitHub", exact: true })); + const connect = screen.getByRole("button", { name: "Connect GitHub", exact: true }); + await waitFor(() => expect(connect).toBeEnabled()); + await user.click(connect); + expect(await screen.findByRole("alert")).toHaveTextContent("Authorization test stopped before redirect"); + expect(starts).toHaveBeenCalledWith("true", "include"); + }); + it.each(["GitHub", "GitLab"] as const)( + "connects %s with a token, saves repositories, and starts a sync", + async (label) => { + const provider = label === "GitHub" ? "github" : "gitlab"; + const apiUrl = provider === "github" ? "https://api.github.com" : "https://gitlab.com/api/v4"; + const saved = vi.fn(); + const closed = vi.fn(); + const writes: { path: string; body: unknown }[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + if (init.method === "PUT" || init.method === "POST") + writes.push({ path, body: init.body ? JSON.parse(String(init.body)) : undefined }); + if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app }); + if (path.endsWith("/repositories")) + return Response.json({ + repositories: [{ name: "org/service", visibility: "private", archived: false }], + has_more: false, + }); + if (path.endsWith("/settings")) { + const request = JSON.parse(String(init.body)) as { repos: string[] }; + const connected = { + ...settings, + id: `saved-${provider}`, + source_provider: provider, + api_url: apiUrl, + has_token: true, + repos: request.repos, + ready: request.repos.length > 0, + }; + return Response.json(connected); + } + if (path.endsWith("/sync")) return Response.json({ running: true }, { status: 202 }); + throw new Error(path); + }), + ); + const user = userEvent.setup(); + render( + , + ); + await user.click(screen.getByRole("button", { name: label, exact: true })); + await user.click(screen.getByRole("button", { name: "Access token", exact: true })); + fireEvent.change(screen.getByLabelText(`${label} access token`), { target: { value: "source-test-token" } }); + await user.click(screen.getByRole("button", { name: "Continue" })); + await user.click(await screen.findByRole("checkbox", { name: /org\/service/ })); + await user.click(screen.getByRole("button", { name: "Save and sync" })); + await waitFor(() => expect(saved).toHaveBeenCalledOnce()); + expect(closed).toHaveBeenCalledOnce(); + expect(writes).toEqual([ + { + path: "/roi-calculator/observed/settings", + body: { + source_provider: provider, + api_url: apiUrl, + token: "source-test-token", + repos: [], + update_interval_minutes: 1440, + }, + }, + { + path: "/roi-calculator/observed/settings", + body: { + connection_id: `saved-${provider}`, + source_provider: provider, + api_url: apiUrl, + repos: ["org/service"], + update_interval_minutes: 1440, + }, + }, + { path: "/roi-calculator/observed/sync", body: undefined }, + ]); + }, + ); + it.each(["GitHub", "GitLab"] as const)( + "starts %s app authorization with a browser cookie and displays provider failures", + async (label) => { + const provider = label.toLowerCase(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + if (String(input).endsWith("/apps")) return Response.json({ github: app, gitlab: app }); + expect(String(input)).toContain(`/oauth/${provider}/start`); + expect(init.credentials).toBe("include"); + expect(init.method).toBe("POST"); + return Response.json({ detail: "Provider unavailable. Try again" }, { status: 502 }); + }), + ); + const user = userEvent.setup(); + render( + , + ); + await user.click(screen.getByRole("button", { name: label, exact: true })); + const button = await screen.findByRole("button", { name: `Connect ${label}`, exact: true }); + await waitFor(() => expect(button).toBeEnabled()); + await user.click(button); + expect(await screen.findByRole("alert")).toHaveTextContent("Provider unavailable. Try again"); + expect(button).toBeEnabled(); + }, + ); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.tsx new file mode 100644 index 00000000000..591b7066e48 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedConnections.tsx @@ -0,0 +1,606 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Github, Gitlab, ArrowLeft, KeyRound } from "lucide-react"; +import { z } from "zod"; +import { apiClient } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog"; +import { + observedSettingsSchema, + repositoryNames, + type ObservedSettings, + type ObservedConnection, +} from "./observedData"; + +const appFields = { + configured: z.boolean(), + can_install: z.boolean().optional().default(false), + api_url: z.string().nullable(), + callback_url: z.string().nullable(), +}; +const appSchema = z.object(appFields); +const appsSchema = z.object({ github: appSchema, gitlab: appSchema }); +const repositoriesSchema = z.object({ + repositories: z.array(z.object({ name: z.string(), visibility: z.string(), archived: z.boolean() })), + has_more: z.boolean(), +}); +const defaultUrl = { github: "https://api.github.com", gitlab: "https://gitlab.com/api/v4" }; + +function preferredConnectionMethod(selected: "app" | "token" | null, configured: boolean | undefined) { + return selected ?? (configured ? "app" : "token"); +} + +function TokenFields({ + label, + provider, + token, + onToken, + apiUrl, + onUrl, + hasToken, +}: { + label: string; + provider: ObservedSettings["source_provider"]; + token: string; + onToken: (value: string) => void; + apiUrl: string; + onUrl: (value: string) => void; + hasToken: boolean; +}) { + return ( + <> + + onToken(event.target.value)} + placeholder={hasToken ? "Leave blank to keep the saved token" : "Optional for public repositories"} + /> +

+ {provider === "github" + ? "Fine-grained token: read access to pull requests, issues, and metadata" + : "Token with read_api scope"} +

+
+ Self-hosted instance + + onUrl(event.target.value)} /> +
+ + ); +} +function AppMessage({ configured, label }: { configured: boolean; label: string }) { + return ( +

+ {configured + ? `You’ll authorize ${label}, then choose repositories` + : `Register the ${label} app in gateway settings, or connect with a token`} +

+ ); +} + +function RepositoryChoices({ + available, + repos, + setRepos, + query, + setQuery, + page, + setPage, +}: { + available: z.infer | null; + repos: string; + setRepos: (value: string) => void; + query: string; + setQuery: (value: string) => void; + page: number; + setPage: (value: number) => void; +}) { + return ( + <> + { + setQuery(event.target.value); + setPage(1); + }} + placeholder="Find repositories…" + /> +
+ {!available && ( +

+ Loading repositories… +

+ )} + {available?.repositories.length === 0 && ( +

No repositories found

+ )} + {available?.repositories + .filter((repo) => !repo.archived) + .map((repo) => ( + + ))} +
+
+ + +
+ + ); +} + +function ConnectionMethod({ + method, + setMethod, + label, + provider, + token, + setToken, + apiUrl, + setApiUrl, + connected, + apps, + busy, + connect, +}: { + method: "app" | "token"; + setMethod: (value: "app" | "token") => void; + label: string; + provider: ObservedSettings["source_provider"]; + token: string; + setToken: (value: string) => void; + apiUrl: string; + setApiUrl: (value: string) => void; + connected: ObservedConnection; + apps: z.infer | null; + busy: boolean; + connect: () => void; +}) { + const connectLabel = method === "app" ? `Connect ${label}` : "Continue"; + const sameSource = connected.source_provider === provider && connected.api_url === apiUrl; + const hasSavedToken = sameSource && connected.has_token && connected.connection_type === "token"; + return ( + <> +
+ + +
+ {method === "token" && ( + + )} + {method === "app" && } + + + ); +} + +type ConnectionStep = "list" | "connect" | "repos"; +const stepTitles = { list: "Connections", connect: "Connect your code", repos: "Choose repositories" }; + +function initialStep(settings: ObservedSettings, afterAuthorization: boolean): ConnectionStep { + if (afterAuthorization) return "repos"; + if (settings.connections?.length) return "list"; + return settings.has_token || settings.ready ? "repos" : "connect"; +} + +function stepDescription(step: ConnectionStep, label: string) { + if (step === "list") return "All selected repositories appear in one report"; + if (step === "connect") return "Connect GitHub and GitLab with an app or access token"; + return `Select ${label} repositories to compare`; +} + +function connectionMethodLabel(entry: ObservedConnection) { + if (entry.connection_type === "app") return "App"; + return entry.has_token ? "Token" : "Public access"; +} + +function ConnectionList({ + connections, + onEdit, + onAdd, +}: { + connections: ObservedConnection[]; + onEdit: (entry: ObservedConnection) => void; + onAdd: () => void; +}) { + return ( + <> + {connections.map((entry) => ( +
+
+

+ {entry.source_provider === "github" ? : } + {entry.source_provider === "github" ? "GitHub" : "GitLab"} +

+

{new URL(entry.api_url).host}

+

+ {entry.repos.length} repositories · {connectionMethodLabel(entry)} +

+
+ +
+ ))} + + + ); +} + +function initialMethod(settings: ObservedSettings) { + return settings.has_token ? settings.connection_type : null; +} + +function hasConnections(settings: ObservedSettings) { + return Boolean(settings.connections?.length); +} + +function canManageApp(connected: ObservedConnection, apps: z.infer | null) { + return ( + connected.connection_type === "app" && connected.source_provider === "github" && Boolean(apps?.github.can_install) + ); +} + +export default function ObservedConnections({ + accessToken, + settings, + onClose, + onSaved, + initialError = "", + afterAuthorization = false, +}: { + accessToken: string; + settings: ObservedSettings; + onClose: () => void; + onSaved: () => void; + initialError?: string; + afterAuthorization?: boolean; +}) { + const [savedSettings, setSavedSettings] = useState(settings); + const [connected, setConnected] = useState(settings); + const [provider, setProvider] = useState(settings.source_provider); + const [apiUrl, setApiUrl] = useState(settings.api_url); + const [selectedMethod, setMethod] = useState<"app" | "token" | null>(initialMethod(settings)); + const [step, setStep] = useState(() => initialStep(settings, afterAuthorization)); + const [token, setToken] = useState(""); + const [repos, setRepos] = useState(settings.repos.join(", ")); + const [apps, setApps] = useState | null>(null); + const method = preferredConnectionMethod(selectedMethod, apps?.[provider].configured); + const [available, setAvailable] = useState | null>(null); + const [query, setQuery] = useState(""); + const [page, setPage] = useState(1); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(initialError); + const label = provider === "github" ? "GitHub" : "GitLab"; + const saveLabel = repositoryNames(repos).length ? "Save and sync" : "Save repositories"; + const manageApp = canManageApp(connected, apps); + const showConnections = hasConnections(savedSettings); + useEffect(() => { + const controller = new AbortController(); + apiClient + .get("/roi-calculator/observed/apps", { accessToken, signal: controller.signal }) + .then((data) => { + if (!controller.signal.aborted) setApps(appsSchema.parse(data)); + }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason)); + }); + return () => controller.abort(); + }, [accessToken]); + useEffect(() => { + if (step !== "repos" || (!connected.has_token && connected.source_provider === "github")) return; + const controller = new AbortController(); + const timer = setTimeout(() => { + apiClient + .get("/roi-calculator/observed/repositories", { + accessToken, + signal: controller.signal, + query: { query, page, connection: connected.id }, + }) + .then((data) => { + if (!controller.signal.aborted) setAvailable(repositoriesSchema.parse(data)); + }) + .catch((reason: unknown) => { + if (!controller.signal.aborted) setError(extractProxyErrorMessage(reason)); + }); + }, 250); + return () => { + clearTimeout(timer); + controller.abort(); + }; + }, [accessToken, step, connected, query, page]); + function selectProvider(value: ObservedSettings["source_provider"]) { + setProvider(value); + const existing = savedSettings.connections?.find( + (entry) => entry.source_provider === value && entry.api_url === defaultUrl[value], + ); + setApiUrl(existing?.api_url ?? defaultUrl[value]); + setConnected( + existing ?? { + ...settings, + source_provider: value, + api_url: defaultUrl[value], + repos: [], + has_token: false, + ready: false, + connection_type: "token", + id: undefined, + }, + ); + setMethod(existing?.connection_type ?? null); + setToken(""); + setError(""); + } + function searchRepositories(value: string) { + setAvailable(null); + setError(""); + setQuery(value); + } + function changePage(value: number) { + setAvailable(null); + setError(""); + setPage(value); + } + async function connect(install = false) { + setBusy(true); + setError(""); + try { + if (method === "app" || install) { + const sameApp = connected.source_provider === provider && connected.connection_type === "app"; + const firstInstallation = provider === "github" && !sameApp && apps?.github.can_install; + const result = z.object({ url: z.string().url() }).parse( + await apiClient.post(`/roi-calculator/observed/oauth/${provider}/start`, { + accessToken, + credentials: "include", + query: { install: install || Boolean(firstInstallation) }, + }), + ); + window.location.assign(result.url); + return; + } + const same = provider === connected.source_provider && apiUrl === connected.api_url; + const keepToken = same && connected.has_token && connected.connection_type === "token"; + const result = observedSettingsSchema.parse( + await apiClient.put("/roi-calculator/observed/settings", { + accessToken, + body: { + connection_id: savedSettings.connections?.find((entry) => entry.id === connected.id)?.id, + source_provider: provider, + api_url: apiUrl, + token: token || (keepToken ? undefined : ""), + repos: same ? connected.repos : [], + update_interval_minutes: connected.update_interval_minutes, + }, + }), + ); + setSavedSettings(result); + setConnected(result); + setToken(""); + setRepos(result.repos.join(", ")); + setAvailable(null); + setStep("repos"); + } catch (reason) { + setError(extractProxyErrorMessage(reason)); + } finally { + setBusy(false); + } + } + async function save() { + setBusy(true); + setError(""); + try { + const result = observedSettingsSchema.parse( + await apiClient.put("/roi-calculator/observed/settings", { + accessToken, + body: { + connection_id: connected.id, + source_provider: connected.source_provider, + api_url: connected.api_url, + repos: repositoryNames(repos), + update_interval_minutes: connected.update_interval_minutes, + }, + }), + ); + if (result.ready) await apiClient.post("/roi-calculator/observed/sync", { accessToken }); + onSaved(); + onClose(); + } catch (reason) { + setError(extractProxyErrorMessage(reason)); + } finally { + setBusy(false); + } + } + function edit(entry: ObservedConnection) { + setConnected(entry); + setProvider(entry.source_provider); + setApiUrl(entry.api_url); + setMethod(entry.connection_type); + setRepos(entry.repos.join(", ")); + setToken(""); + setQuery(""); + setPage(1); + setAvailable(null); + setError(""); + setStep("repos"); + } + return ( + { + if (!open) onClose(); + }} + > + + + {stepTitles[step]} + {stepDescription(step, label)} + +
+ {step === "list" && ( + { + selectProvider( + savedSettings.connections?.some((entry) => entry.source_provider === "github") ? "gitlab" : "github", + ); + setStep("connect"); + }} + /> + )} + {step === "connect" && ( + <> + {showConnections && ( + + )} +
+ {(["github", "gitlab"] as const).map((value) => ( + + ))} +
+ connect()} + /> + + )} + {step === "repos" && ( + <> + {showConnections && ( + + )} + + {manageApp && ( + + )} + + setRepos(event.target.value)} + placeholder={ + provider === "github" ? "owner/repo, owner/another-repo" : "group/project, group/subgroup/project" + } + /> + {(connected.has_token || connected.source_provider === "gitlab") && ( + <> + + + )} + + + )} + {error && ( +

+ {error} +

+ )} +
+ + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedDetails.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedDetails.tsx new file mode 100644 index 00000000000..02d60a05029 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedDetails.tsx @@ -0,0 +1,232 @@ +"use client"; + +import { useState } from "react"; +import { ExternalLink, GitPullRequest } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetDescription } from "@/components/ui/sheet"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { + dateRange, + changeTerms, + duration, + money, + number, + recordedBranches, + type Comparison, + type ObservedPerson, + type ObservedPull, + type ObservedSnapshot, + type Period, +} from "./observedData"; + +export function PullList({ + pulls, + provider, +}: { + pulls: ObservedPull[]; + provider: ObservedSnapshot["source_provider"]; +}) { + const terms = changeTerms(provider); + const [query, setQuery] = useState(""); + const [limit, setLimit] = useState(20); + const filtered = pulls.filter((pull) => + `${pull.number} ${pull.title} ${pull.author} ${pull.repo} ${pull.source_repo} ${pull.source_branch}` + .toLowerCase() + .includes(query.toLowerCase()), + ); + return ( +
+ { + setQuery(event.target.value); + setLimit(20); + }} + className="max-w-sm" + /> + + + + {terms.requests} + Author + Opened to merged + Tagged spend + + + + {filtered.slice(0, limit).map((pull) => ( + + + + + + #{pull.number} + {pull.title} + {pull.repo} + + + + + + {pull.author || "Deleted author"} + {pull.agent && ( + + Agent + + )} + + {duration(pull.merge_hours)} + + {money(pull.branch_cost.spend)} + + + ))} + +
+

+ Elapsed time from opening to merge, not engineering effort or time saved +

+ {filtered.length === 0 && ( +

+ {query ? `No ${terms.lower} match this search` : `No ${terms.lower} in this period`} +

+ )} +
+ + {number(Math.min(limit, filtered.length))} of {number(filtered.length)} {terms.lower} + + {limit < filtered.length && ( + + )} +
+
+ ); +} + +export function PersonDetails({ + person, + snapshot, + comparison, + onClose, + onEdit, +}: { + person: ObservedPerson; + snapshot: ObservedSnapshot; + comparison: Comparison; + onClose: () => void; + onEdit?: () => void; +}) { + const terms = changeTerms(snapshot.source_provider); + const [period, setPeriod] = useState("current"); + const current = person.periods.current; + const baseline = person.periods[comparison]; + const urls = new Set(person.periods[period].pr_urls); + const pulls = snapshot.pulls[period].filter((pull) => urls.has(pull.url)); + return ( + { + if (!open) onClose(); + }} + > + + + {person.name} + + {person.email} · {person.logins.join(", ")} + + + {onEdit && ( + + )} +
+
+

Merged {terms.plural}

+

{number(current.merged_prs)}

+

{number(baseline.merged_prs)} in comparison

+
+
+

Recorded spend

+

+ {money(current.spend_observation === "no_records" ? null : current.gateway_recorded_spend)} +

+

Gateway only

+
+
+

Spend / matched {terms.singular}

+

{money(current.recorded_spend_per_attributed_pr)}

+

Period average

+
+
+
+

Merged {terms.plural}

+
+ + +
+
+

{dateRange(snapshot.periods[period].window)} · UTC

+ + + + ); +} + +export function BranchSpend({ snapshot }: { snapshot: ObservedSnapshot }) { + const rows = recordedBranches(snapshot); + return ( +
+ + + + Repository + Branch + Requests + Tagged spend + + + + {rows.map((row) => ( + + {row.repo} + {row.branch} + {number(row.requests)} + {money(row.spend)} + + ))} + +
+ {rows.length === 0 && ( +

No tagged branch spend in this period

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.integration.test.tsx new file mode 100644 index 00000000000..baab6eee876 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.integration.test.tsx @@ -0,0 +1,305 @@ +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import ObservedROIView from "./ObservedROIView"; +import type { ObservedSettings, ObservedSnapshot, ObservedStatus } from "./observedData"; + +const settings: ObservedSettings = { + source_provider: "gitlab", + api_url: "https://gitlab.com/api/v4", + repos: ["org/service"], + has_token: true, + connection_type: "app", + update_interval_minutes: 1440, + ready: true, +}; +const idle: ObservedStatus = { + running: false, + phase: "complete", + stage: "", + done: 0, + total: 0, + error: null, + finished_at: null, +}; +const period = { + window: { start: "2026-09-01", end: "2026-09-28" }, + merged_prs: 1, + median_merge_hours: 16 / 3600, + human_authored: 1, + agent_authored: 0, + missing_author: 0, + agents_without_requester: 0, + matched_internal_prs: 0, + new_bug_labeled_issues: 0, + new_regression_labeled_issues: 0, + explicitly_titled_revert_prs: 0, + matched_users_recorded_spend: 0, + spend_observation: "no_records" as const, + human_summary: { median_merge_hours: 16 / 3600 }, +}; +const report: ObservedSnapshot = { + source_provider: "gitlab", + repos: ["org/service"], + unmatched_logins: [], + unlinked_branches: [], + captured_at: "2026-09-29T00:00:00Z", + periods: { current: period, previous: period, last_year: period }, + people: [], + pulls: { current: [], previous: [], last_year: [] }, +}; +const app = { configured: true, api_url: null, callback_url: null }; + +afterEach(() => { + vi.unstubAllGlobals(); + window.history.replaceState(null, "", "/"); +}); + +describe("observed ROI dashboard", () => { + it("previews every sample view before setup, changes sample periods without writes, and exits back to setup", async () => { + const requests = vi.fn(async (input: string, _init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + const disconnected = { ...settings, ready: false, repos: [], has_token: false }; + if (path.endsWith("/settings")) return Response.json(disconnected); + if (path.endsWith("/report")) return Response.json({ report: null }); + if (path.endsWith("/sync")) return Response.json(idle); + throw new Error(path); + }); + vi.stubGlobal("fetch", requests); + const user = userEvent.setup(); + render(); + expect(await screen.findByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument(); + expect(screen.queryByText("Ready to sync")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Preview sample report" })); + expect(screen.getByRole("status")).toHaveTextContent("You’re viewing demo data"); + expect(screen.getByRole("tab", { name: "Engineers 3", selected: true })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Connections" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Link accounts" })).not.toBeInTheDocument(); + expect(window.location.search).toBe("?demo=1"); + await user.click(screen.getByRole("button", { name: "View Alex Rivera's merged changes" })); + expect(await screen.findByRole("dialog", { name: "Alex Rivera" })).toHaveTextContent("alex-demo@example.com"); + expect(screen.getByRole("heading", { name: "Merged changes" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit linked accounts" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Close" })); + await user.click(screen.getByRole("tab", { name: "Merged changes" })); + expect(screen.getByRole("img", { name: /Merged changes by week/ })).toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: "Quality" })); + expect(screen.getByText("New regression-labeled issues")).toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: "Branch spend" })); + expect(screen.getAllByText(/feature\/sample-/).length).toBeGreaterThan(0); + await user.click(screen.getByRole("combobox", { name: "Reporting period" })); + await user.click(await screen.findByRole("option", { name: "Last 7 days" })); + expect(screen.getByRole("combobox", { name: "Reporting period" })).toHaveTextContent("Last 7 days"); + await user.click(screen.getByRole("combobox", { name: "Comparison period" })); + await user.click(await screen.findByRole("option", { name: "vs. same period last year" })); + expect(screen.getByRole("combobox", { name: "Comparison period" })).toHaveTextContent("vs. same period last year"); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(screen.getByRole("heading", { name: "Connect your repositories" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Connect GitHub or GitLab" })).toBeEnabled(); + expect(window.location.search).toBe(""); + expect(requests.mock.calls.every(([, init]) => init.method === "GET")).toBe(true); + }); + + it("keeps live sync and its report intact when entering and exiting the demo", async () => { + const requests = vi.fn(async (input: string, _init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + if (path.endsWith("/settings")) return Response.json(settings); + if (path.endsWith("/report")) return Response.json({ report }); + if (path.endsWith("/sync")) return Response.json({ ...idle, running: true, stage: "Reading changes" }); + throw new Error(path); + }); + vi.stubGlobal("fetch", requests); + window.history.replaceState(null, "", "/roi-calculator/?from=review#report"); + const user = userEvent.setup(); + render(); + expect(await screen.findByRole("button", { name: "Cancel sync" })).toBeEnabled(); + await user.click(screen.getByRole("button", { name: "Preview sample report" })); + expect(screen.queryByRole("button", { name: "Cancel sync" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "2 repositories" })).toBeInTheDocument(); + expect(window.location.search).toBe("?from=review&demo=1"); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(screen.getByRole("button", { name: "Cancel sync" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "1 repository" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Merge requests", selected: true })).toBeInTheDocument(); + expect(window.location.search).toBe("?from=review"); + expect(window.location.hash).toBe("#report"); + expect(requests.mock.calls.every(([, init]) => init.method === "GET")).toBe(true); + }); + + it.each(["failed", "pending"])("opens a demo URL even when live requests are %s", async (state) => { + window.history.replaceState(null, "", "/roi-calculator/?demo=1"); + const pending = Promise.withResolvers(); + vi.stubGlobal( + "fetch", + vi.fn(() => (state === "failed" ? Promise.reject(new Error("Live data unavailable")) : pending.promise)), + ); + const user = userEvent.setup(); + render(); + expect(screen.getByRole("status")).toHaveTextContent("You’re viewing demo data"); + expect(screen.getByText("Alex Rivera")).toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(screen.queryByText("Alex Rivera")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); + expect(window.location.search).toBe(""); + if (state === "failed") expect(await screen.findByRole("alert")).toHaveTextContent("Live data unavailable"); + }); + + it("retries failures, keeps the report during cancellation, and refreshes after completion", async () => { + let status: ObservedStatus = { ...idle, phase: "error", error: "Provider temporarily unavailable" }; + let completeOnPoll = false; + let currentReport = { ...report, repos: ["org/service", "org/docs"] }; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + if (path.endsWith("/settings")) return Response.json(settings); + if (path.endsWith("/report")) return Response.json({ report: currentReport }); + if (path.endsWith("/sync")) { + if (init.method === "POST") status = { ...idle, running: true, phase: "pulls" }; + if (init.method === "DELETE") status = { ...idle, phase: "cancelled" }; + if (init.method === "GET" && completeOnPoll) { + status = { ...idle, finished_at: "2026-09-29T00:01:00Z" }; + currentReport = { ...report, repos: ["org/updated"] }; + } + return Response.json(status); + } + throw new Error(path); + }), + ); + const user = userEvent.setup(); + render(); + expect(await screen.findByRole("tab", { name: "Merge requests", selected: true })).toBeInTheDocument(); + await user.click(screen.getByRole("tab", { name: "Quality" })); + expect(screen.getByRole("alert")).toHaveTextContent("Provider temporarily unavailable"); + await user.click(screen.getByRole("button", { name: "Retry" })); + await user.click(await screen.findByRole("button", { name: "Cancel sync" })); + expect(await screen.findByRole("button", { name: "Sync now" })).toBeEnabled(); + expect(screen.queryByText("org/service")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "2 repositories" })); + const repositories = await screen.findByRole("dialog", { name: "Repositories" }); + expect(within(repositories).getByText("org/service")).toBeInTheDocument(); + expect(within(repositories).getByText("org/docs")).toBeInTheDocument(); + await user.keyboard("{Escape}"); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Repositories" })).not.toBeInTheDocument()); + await user.click(screen.getByRole("button", { name: "Sync now" })); + expect(await screen.findByRole("button", { name: "Cancel sync" })).toBeEnabled(); + completeOnPoll = true; + await user.click(await screen.findByRole("button", { name: "1 repository" }, { timeout: 4000 })); + expect( + await within(screen.getByRole("dialog", { name: "Repositories" })).findByText("org/updated"), + ).toBeInTheDocument(); + await user.keyboard("{Escape}"); + expect(screen.getByRole("tab", { name: "Quality", selected: true })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Sync now" })).toBeEnabled(); + }); + + it("syncs the selected range and labels its equal-length comparison", async () => { + let currentReport = report; + const requested: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const url = new URL(input, "http://localhost"); + if (url.pathname.endsWith("/settings")) return Response.json(settings); + if (url.pathname.endsWith("/report")) return Response.json({ report: currentReport }); + if (url.pathname.endsWith("/sync")) { + if (init.method === "POST") { + requested.push(url.searchParams.get("days") ?? ""); + currentReport = { + ...report, + periods: { + ...report.periods, + current: { ...period, window: { start: "2026-09-22", end: "2026-09-28" } }, + previous: { ...period, window: { start: "2026-09-15", end: "2026-09-21" } }, + }, + }; + } + return Response.json(idle); + } + throw new Error(url.pathname); + }), + ); + const user = userEvent.setup(); + render(); + await user.click(await screen.findByRole("combobox", { name: "Reporting period" })); + await user.click(await screen.findByRole("option", { name: "Last 7 days" })); + await waitFor(() => expect(requested).toEqual(["7"])); + expect(await screen.findByText(/Comparing with Sep 15.*Sep 21/)).toBeInTheDocument(); + expect(screen.getByRole("combobox", { name: "Reporting period" })).toHaveTextContent("Last 7 days"); + expect(screen.getByRole("combobox", { name: "Comparison period" })).toHaveTextContent("vs. previous period"); + }); + + it("shows a successful empty repository without a setup prompt or invented durations", async () => { + const emptyPeriod = { + ...period, + merged_prs: 0, + human_authored: 0, + median_merge_hours: null, + human_summary: { median_merge_hours: null }, + }; + const empty = { ...report, periods: { current: emptyPeriod, previous: emptyPeriod, last_year: emptyPeriod } }; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string) => { + const path = new URL(input, "http://localhost").pathname; + if (path.endsWith("/settings")) return Response.json(settings); + if (path.endsWith("/report")) return Response.json({ report: empty }); + if (path.endsWith("/sync")) return Response.json(idle); + throw new Error(path); + }), + ); + render(); + expect(await screen.findByRole("heading", { name: "No merged changes yet" })).toBeInTheDocument(); + expect(screen.getByText("No merges")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Sync now" })).toBeEnabled(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Connect your repositories" })).not.toBeInTheDocument(); + expect(screen.queryByText("0h")).not.toBeInTheDocument(); + }); + + it.each([ + { query: "connected=gitlab", alerts: [] }, + { query: "connection_failed=1", alerts: ["Connection failed or expired. Try again or use a token"] }, + { query: "connection_cancelled=1", alerts: ["Connection cancelled. Choose an app or token to try again"] }, + ])("resumes setup after $query and refreshes saved changes after closing", async ({ query, alerts }) => { + window.history.replaceState(null, "", `/roi-calculator/?${query}`); + let currentSettings = settings; + vi.stubGlobal( + "fetch", + vi.fn(async (input: string, init: RequestInit) => { + const path = new URL(input, "http://localhost").pathname; + if (path.endsWith("/apps")) return Response.json({ github: app, gitlab: app }); + if (path.endsWith("/repositories")) return Response.json({ repositories: [], has_more: false }); + if (path.endsWith("/settings")) { + if (init.method === "PUT") currentSettings = { ...settings, repos: ["org/changed"] }; + return Response.json(currentSettings); + } + if (path.endsWith("/report")) return Response.json({ report: { ...report, repos: currentSettings.repos } }); + if (path.endsWith("/sync")) + return init.method === "POST" + ? Response.json({ detail: "Provider unavailable" }, { status: 502 }) + : Response.json(idle); + throw new Error(path); + }), + ); + const user = userEvent.setup(); + render(); + const dialog = await screen.findByRole("dialog", { name: "Choose repositories" }); + expect( + within(dialog) + .queryAllByRole("alert") + .map((alert) => alert.textContent), + ).toEqual(alerts); + expect(window.location.search).toBe(""); + fireEvent.change(within(dialog).getByLabelText("Repositories"), { target: { value: "org/changed" } }); + await user.click(within(dialog).getByRole("button", { name: "Save and sync" })); + expect(await within(dialog).findByRole("alert")).toHaveTextContent("Provider unavailable"); + await user.click(within(dialog).getByRole("button", { name: "Close" })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + await user.click(await screen.findByRole("button", { name: "1 repository" })); + expect( + await within(screen.getByRole("dialog", { name: "Repositories" })).findByText("org/changed"), + ).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.tsx new file mode 100644 index 00000000000..29199b67f22 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedROIView.tsx @@ -0,0 +1,253 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Link2, RefreshCw } from "lucide-react"; +import { apiClient } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { Page } from "@/components/shared/Page"; +import { PageHeader, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; +import { DemoNotice } from "@/components/shared/DemoNotice"; +import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; +import ObservedConnections from "./ObservedConnections"; +import ObservedReport from "./ObservedReport"; +import { useObservedReport, type ObservedViewData } from "./useObservedReport"; +import { syncMessage, type ObservedSnapshot } from "./observedData"; +import { createObservedDemo } from "./observedDemo"; + +function SyncActions({ + data, + error, + busy, + readOnly, + compact = false, + onSync, + onRetry, +}: { + data: ObservedViewData | null; + error: string; + busy: boolean; + readOnly: boolean; + compact?: boolean; + onSync: (cancel: boolean) => void; + onRetry: () => void; +}) { + const message = error || data?.status.error; + const statusMessage = data ? syncMessage(data.status, data.report) : ""; + const canSync = data?.settings.ready && !readOnly; + if (!message && !statusMessage && !canSync) return null; + return ( +
+ {message && ( +
+ {message} + +
+ )} + {data && ( +
+ {statusMessage && ( + + {statusMessage} + + )} + {!readOnly && data.settings.ready && ( + + )} +
+ )} +
+ ); +} + +function EmptyReport({ + data, + readOnly, + onConnect, +}: { + data: ObservedViewData; + readOnly: boolean; + onConnect: () => void; +}) { + function title() { + if (data.status.running) return "Reading repository activity"; + return data.settings.ready ? "Ready for your first report" : "Connect your repositories"; + } + return ( +
+

{title()}

+

+ {data.status.running + ? "Your report will appear here when the first sync finishes" + : "Compare merged changes, issue trends, and recorded AI spend across your team"} +

+ {!readOnly && !data.status.running && ( + + )} +
+ ); +} + +export default function ObservedROIView({ + accessToken, + isViewOnly = false, +}: { + accessToken: string; + isViewOnly?: boolean; +}) { + const { data, error, refresh } = useObservedReport(accessToken); + const [returned] = useState(() => new URLSearchParams(typeof window === "undefined" ? "" : window.location.search)); + const [sample, setSample] = useState(() => + returned.get("demo") === "1" ? createObservedDemo(28) : null, + ); + const [connections, setConnections] = useState( + ["github", "gitlab"].includes(returned.get("connected") ?? "") || + returned.has("connection_cancelled") || + returned.has("connection_failed"), + ); + const [connectionError, setConnectionError] = useState(() => { + if (returned.has("connection_failed")) return "Connection failed or expired. Try again or use a token"; + if (returned.has("connection_cancelled")) return "Connection cancelled. Choose an app or token to try again"; + return ""; + }); + const [afterAuthorization, setAfterAuthorization] = useState(Boolean(returned.get("connected"))); + const [busy, setBusy] = useState(false); + const [actionError, setActionError] = useState(""); + useEffect(() => { + const url = new URL(window.location.href); + url.searchParams.delete("connected"); + url.searchParams.delete("connection_cancelled"); + url.searchParams.delete("connection_failed"); + window.history.replaceState(window.history.state, "", url); + }, []); + function previewSample(enabled: boolean) { + const url = new URL(window.location.href); + if (enabled) url.searchParams.set("demo", "1"); + else url.searchParams.delete("demo"); + window.history.replaceState(window.history.state, "", url); + setSample(enabled ? createObservedDemo(28) : null); + } + function closeConnections() { + setConnections(false); + setConnectionError(""); + setAfterAuthorization(false); + refresh(); + } + async function sync(cancel: boolean, days?: number) { + setBusy(true); + setActionError(""); + try { + if (cancel) await apiClient.delete("/roi-calculator/observed/sync", { accessToken }); + else await apiClient.post("/roi-calculator/observed/sync", { accessToken, query: { days } }); + refresh(); + } catch (reason) { + setActionError(extractProxyErrorMessage(reason)); + } finally { + setBusy(false); + } + } + function retry() { + if (error || isViewOnly || !data?.settings.ready) refresh(); + else void sync(data.status.running); + } + if (sample) { + return ( + setConnections(true)} + actions={null} + notice={ previewSample(false)} />} + syncing={false} + onPeriod={(days) => setSample(createObservedDemo(days))} + /> + ); + } + const previewButton = ( + + ); + const actions = ( + <> + {previewButton} + + + ); + const content = data?.report ? ( + setConnections(true)} + actions={actions} + syncing={busy || data.status.running} + onPeriod={isViewOnly ? undefined : (days) => void sync(false, days)} + /> + ) : ( + + +
+ ROI Calculator + {previewButton} +
+ Are we shipping more, with fewer bugs, at a better cost? + + + {!data && !error && ( + <> + + + + )} + {data && setConnections(true)} />} + + ); + const showConnections = connections && data && !isViewOnly; + return ( + <> + {content} + {showConnections && ( + + )} + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedReport.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedReport.tsx new file mode 100644 index 00000000000..d68750480cc --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ObservedReport.tsx @@ -0,0 +1,611 @@ +"use client"; + +import { useState } from "react"; +import { ArrowDown, ArrowUp, CalendarDays, ChevronDown, ChevronRight, Link2, Search, Users } from "lucide-react"; +import { Page, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; +import { PageHeader, PageHeaderTitle } from "@/components/shared/PageHeader"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Tabs, TabsContent } from "@/components/ui/tabs"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import ObservedAccounts from "./ObservedAccounts"; +import { BranchSpend, PersonDetails, PullList } from "./ObservedDetails"; +import { + change, + dateRange, + changeTerms, + duration, + money, + number, + visiblePeople, + weeklyMerges, + type Comparison, + type ObservedPerson, + type ObservedSnapshot, + type PeopleSort, +} from "./observedData"; + +function Delta({ + current, + baseline, + neutral = false, +}: { + current: number | null; + baseline: number | null; + neutral?: boolean; +}) { + const delta = current === null || baseline === null ? null : change(current, baseline); + if (delta === null) return No baseline; + const Icon = delta >= 0 ? ArrowUp : ArrowDown; + return ( + = 0 ? "increase" : "decrease"}`} + className={`inline-flex items-center gap-1 text-xs tabular-nums ${neutral ? "text-muted-foreground" : "text-foreground"}`} + > + + {number(Math.abs(delta))}% + + ); +} + +function Metric({ + label, + value, + detail, + current, + baseline, +}: { + label: string; + value: string; + detail: string; + current?: number | null; + baseline?: number | null; +}) { + return ( +
+
{label}
+
+ {value} + {current !== undefined && baseline !== undefined && } +
+

{detail}

+
+ ); +} + +function ShippingTrend({ snapshot, comparison }: { snapshot: ObservedSnapshot; comparison: Comparison }) { + const terms = changeTerms(snapshot.source_provider); + const current = weeklyMerges(snapshot, "current"); + const baseline = weeklyMerges(snapshot, comparison); + const max = Math.max(1, ...current, ...baseline); + return ( +
+
+

Shipping activity

+
+ + + Current period + + + + {comparison === "previous" ? "Previous period" : "Last year"} + +
+
+
+ {current.map((value, week) => ( +
+
+
+ + {baseline[week]} + +
+
+ {value} +
+
+

W{week + 1}

+
+ ))} +
+
+ ); +} + +function PeopleTable({ + snapshot, + comparison, + onSelect, +}: { + snapshot: ObservedSnapshot; + comparison: Comparison; + onSelect: (person: ObservedPerson) => void; +}) { + const terms = changeTerms(snapshot.source_provider); + const [query, setQuery] = useState(""); + const [sort, setSort] = useState("merged"); + const people = visiblePeople(snapshot.people, query, sort); + return ( +
+
+
+ + setQuery(event.target.value)} + className="pl-9" + /> +
+
+ + {people.length} {people.length === 1 ? "engineer" : "engineers"} + + +
+
+
+ + + + Engineer + Merged {terms.plural} + Authored / agent + + {comparison === "previous" ? "vs. previous" : "vs. last year"} + + Median merge + Recorded spend + Spend / {terms.singular} + + Details + + + + + {people.map((person) => { + const current = person.periods.current; + const baseline = person.periods[comparison]; + return ( + + + + + {number(current.merged_prs)} + +
+
+ + +
+ + {current.direct_authored}/{current.declared_agent_owned} + +
+ + + + ({baseline.merged_prs}) + + {duration(current.median_merge_hours)} + + {money(current.spend_observation === "no_records" ? null : current.gateway_recorded_spend)} + + + {money(current.recorded_spend_per_attributed_pr)} + + + + + + ); + })} + +
+ {people.length === 0 && ( +
+ {query ? `No engineers match “${query}”` : "Link accounts to see your engineers"} +
+ )} +
+

+ + + Authored + + + + Agent, explicit requester + + Spend / {terms.singular} is recorded period spend divided by matched {terms.plural} +

+
+ ); +} + +function Quality({ snapshot, comparison }: { snapshot: ObservedSnapshot; comparison: Comparison }) { + const terms = changeTerms(snapshot.source_provider); + const current = snapshot.periods.current; + const baseline = snapshot.periods[comparison]; + const rows = [ + { + label: "New bug-labeled issues", + current: current.new_bug_labeled_issues, + baseline: baseline.new_bug_labeled_issues, + detail: "Opened during the period, with bug or kind:bug labels at collection", + }, + { + label: "New regression-labeled issues", + current: current.new_regression_labeled_issues, + baseline: baseline.new_regression_labeled_issues, + detail: "Opened during the period and labeled as regressions", + }, + { + label: `Revert-titled ${terms.plural}`, + current: current.explicitly_titled_revert_prs, + baseline: baseline.explicitly_titled_revert_prs, + detail: `Merged ${terms.plural} whose titles explicitly indicate a revert`, + }, + ]; + return ( +
+
+ + + + Repository signal + Current + Comparison + Change + + + + {rows.map((row) => ( + + +

{row.label}

+

{row.detail}

+ + {number(row.current)} + {number(row.baseline)} + + + + + ))} + +
+
+

+ These signals help check whether more shipping comes with more bugs. Labels and revert titles are incomplete + proxies; they do not establish a change-failure rate or attribute bugs to an engineer. +

+
+ ); +} + +function costPerChange(period: ObservedSnapshot["periods"]["current"]) { + if (period.spend_observation !== "records_present" || period.matched_internal_prs === 0) return null; + return period.matched_users_recorded_spend / period.matched_internal_prs; +} + +export default function ObservedReport({ + snapshot, + accessToken, + readOnly, + onRefresh, + onConnect, + actions, + notice, + syncing, + onPeriod, +}: { + snapshot: ObservedSnapshot; + accessToken: string; + readOnly: boolean; + onRefresh: () => void; + onConnect: () => void; + actions: React.ReactNode; + notice?: React.ReactNode; + syncing: boolean; + onPeriod?: (days: number) => void; +}) { + const [comparison, setComparison] = useState("previous"); + const [activeTab, setActiveTab] = useState( + snapshot.people.length && snapshot.periods.current.merged_prs > 0 ? "people" : "pulls", + ); + const [accountEmail, setAccountEmail] = useState(null); + const [personEmail, setPersonEmail] = useState(null); + const person = snapshot.people.find((entry) => entry.email === personEmail) ?? null; + const terms = changeTerms(snapshot.source_provider); + const current = snapshot.periods.current; + const baseline = snapshot.periods[comparison]; + const days = Math.round((Date.parse(current.window.end) - Date.parse(current.window.start)) / 86400000) + 1; + const rangeOptions = [...new Set([7, 28, 90, days])].sort((a, b) => a - b); + const cost = costPerChange(current); + const baselineCost = costPerChange(baseline); + return ( + + + ROI Calculator +
+ {actions} + {!readOnly && ( + + )} +
+ + {notice} +
+ + }> + {number(snapshot.repos.length)} {snapshot.repos.length === 1 ? "repository" : "repositories"} + + + + Repositories +
    + {snapshot.repos.map((repo) => ( +
  • {repo}
    • + ))} +
+ + +
+ + + + {dateRange(current.window)} + + +
+
+
+ + + + +
+ +
+ + + Engineers {snapshot.people.length} + + {terms.requests} + Quality + Branch spend + + {!readOnly && ( + + )} +
+ + setPersonEmail(selected.email)} + /> + + + {current.merged_prs > 0 || baseline.merged_prs > 0 ? ( +
+ +
+
+

Behind the numbers

+

+ {number(current.agent_authored)} of {number(current.merged_prs)} {terms.plural} were authored by + agents or bots. +

+

+ Human-authored median merge time:{" "} + + {duration(current.human_summary.median_merge_hours)} + + , compared with {duration(baseline.human_summary.median_merge_hours)}. +

+
+
+ + {number(current.agents_without_requester)} agent {terms.plural} have no requester + +
+
+
+ ) : ( +
+

No merged changes yet

+

+ Your repositories are connected. New activity will appear after the next sync +

+
+ )} + +

+ All repository {terms.plural}, including agent work without a known requester +

+ + + + + + + + + +
+ + {number(current.matched_internal_prs)} {terms.plural} matched to {snapshot.people.length} engineers ·{" "} + {number(current.agents_without_requester)} agent {terms.plural} without a requester + + Comparing with {dateRange(baseline.window)} · All dates UTC + Spend recorded by this gateway · Merge time is elapsed time, not effort +
+ {accountEmail !== null && ( + setAccountEmail(null)} + onSaved={onRefresh} + /> + )} + {person && ( + setPersonEmail(null)} + onEdit={ + readOnly + ? undefined + : () => { + setAccountEmail(person.email); + setPersonEmail(null); + } + } + /> + )} + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.test.ts new file mode 100644 index 00000000000..57c37af900a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import { + change, + duration, + money, + visiblePeople, + weeklyMerges, + type ObservedPerson, + type ObservedSnapshot, +} from "./observedData"; + +const period = (merged: number, spend: number | null) => ({ + merged_prs: merged, + prs_per_week: merged / 4, + median_merge_hours: null, + direct_authored: merged, + declared_agent_owned: 0, + gateway_recorded_spend: spend ?? 0, + recorded_spend_per_attributed_pr: spend !== null && merged > 0 ? spend / merged : null, + spend_observation: spend === null ? ("no_records" as const) : ("records_present" as const), + pr_urls: [], +}); +const person = (name: string, merged: number, spend: number | null): ObservedPerson => ({ + name, + email: `${name}@example.test`, + logins: [`old-${name}`], + periods: { current: period(merged, spend), previous: period(0, null), last_year: period(0, null) }, +}); + +describe("observed ROI metrics", () => { + it("does not claim infinite growth when the baseline is missing", () => { + expect(change(12, 0)).toBeNull(); + expect(change(0, 0)).toBeNull(); + expect(change(15, 10)).toBe(50); + expect(change(0, 10)).toBe(-100); + }); + + it("distinguishes missing cost, measured zero, and small nonzero spend", () => { + expect(money(null)).toBe("Unavailable"); + expect(money(0)).toBe("$0.00"); + expect(money(0.001)).toBe("<$0.01"); + expect(money(1.235)).toBe("$1.24"); + expect(duration(null)).toBe("Unavailable"); + }); + + it("searches historical identities and sorts without changing the source", () => { + const people = [person("Ari", 2, 8), person("Bea", 10, 0), person("Cam", 0, null)]; + expect(visiblePeople(people, " OLD-ARI ", "merged").map((row) => row.name)).toEqual(["Ari"]); + expect(visiblePeople(people, "", "merged").map((row) => row.name)).toEqual(["Bea", "Ari", "Cam"]); + expect(visiblePeople(people, "", "cost").map((row) => row.name)).toEqual(["Ari", "Bea", "Cam"]); + expect(people.map((row) => row.name)).toEqual(["Ari", "Bea", "Cam"]); + expect(visiblePeople(people, "missing", "name")).toEqual([]); + }); + + it("keeps short elapsed merge times from rounding to zero hours", () => { + expect(duration(null)).toBe("Unavailable"); + expect(duration(0)).toBe("0m"); + expect(duration(16 / 3600)).toBe("<1m"); + expect(duration(59 / 3600)).toBe("<1m"); + expect(duration(1 / 60)).toBe("1m"); + expect(duration(79 / 3600)).toBe("1.3m"); + expect(duration(140 / 3600)).toBe("2.3m"); + expect(duration(0.5)).toBe("30m"); + expect(duration(1)).toBe("1h"); + expect(duration(3.82)).toBe("3.8h"); + }); + + it.each([7, 28, 90])("includes every day of a %s-day range in its weekly chart", (days) => { + const start = Date.parse("2026-01-01T00:00:00Z"); + const atDay = (day: number) => new Date(start + day * 86400000).toISOString(); + const pulls = Array.from({ length: days + 1 }, (_, day) => ({ merged_at: atDay(day) })); + const snapshot = { + periods: { current: { window: { start: "2026-01-01", end: atDay(days - 1).slice(0, 10) } } }, + pulls: { current: pulls }, + } as ObservedSnapshot; + const weeks = weeklyMerges(snapshot, "current"); + expect(weeks).toHaveLength(Math.ceil(days / 7)); + expect(weeks.reduce((sum, count) => sum + count, 0)).toBe(days); + expect(weeks.at(-1)).toBe(days % 7 || 7); + }); + + it("aligns comparisons to their own UTC windows and counts each boundary once", () => { + const currentStart = "2026-01-01"; + const previousStart = "2025-12-04"; + const pulls = [ + "2026-01-01T00:00:00Z", + "2026-01-07T23:59:59Z", + "2026-01-08T00:00:00Z", + "2026-01-28T23:59:59Z", + "2026-01-29T00:00:00Z", + ].map((merged_at, number) => ({ + number, + merged_at, + title: "Example", + url: "https://github.com/example/repo/pull/1", + author: "ari", + agent: false, + merge_hours: 1, + })); + const snapshot = { + periods: { + current: { window: { start: currentStart, end: "2026-01-28" } }, + previous: { window: { start: previousStart, end: "2025-12-31" } }, + }, + pulls: { current: pulls, previous: [{ ...pulls[0], merged_at: `${previousStart}T00:00:00Z` }] }, + } as ObservedSnapshot; + expect(weeklyMerges(snapshot, "current")).toEqual([2, 1, 0, 1]); + expect(weeklyMerges(snapshot, "previous")).toEqual([1, 0, 0, 0]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.ts b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.ts new file mode 100644 index 00000000000..3dbd115421c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedData.ts @@ -0,0 +1,212 @@ +import { z } from "zod"; + +const windowSchema = z.object({ start: z.string(), end: z.string() }); +const personPeriodFields = { + merged_prs: z.number(), + prs_per_week: z.number(), + median_merge_hours: z.number().nullable(), + direct_authored: z.number(), + declared_agent_owned: z.number(), + gateway_recorded_spend: z.number(), + recorded_spend_per_attributed_pr: z.number().nullable(), + spend_observation: z.enum(["records_present", "no_records"]), + pr_urls: z.array(z.string()), +}; +const personPeriodSchema = z.object(personPeriodFields); +const periodFields = { + window: windowSchema, + merged_prs: z.number(), + median_merge_hours: z.number().nullable(), + human_authored: z.number(), + agent_authored: z.number(), + missing_author: z.number(), + agents_without_requester: z.number(), + matched_internal_prs: z.number(), + new_bug_labeled_issues: z.number().nullable(), + new_regression_labeled_issues: z.number().nullable(), + explicitly_titled_revert_prs: z.number(), + matched_users_recorded_spend: z.number(), + spend_observation: z.enum(["records_present", "no_records"]), + human_summary: z.object({ median_merge_hours: z.number().nullable() }), +}; +const periodSchema = z.object(periodFields); +const periods = (schema: T) => z.object({ current: schema, previous: schema, last_year: schema }); + +const personFields = { + name: z.string(), + email: z.string(), + logins: z.array(z.string()), + periods: periods(personPeriodSchema), + accounts: z.array(z.object({ connection_id: z.string(), login: z.string() })).optional(), +}; +const branchCostFields = { + repo: z.string(), + branch: z.string(), + spend: z.number().nullable(), + requests: z.number(), + status: z.enum(["matched", "unattributed", "ambiguous", "unavailable"]), +}; +const branchSpendFields = { repo: z.string(), branch: z.string(), spend: z.number(), requests: z.number() }; +const pullFields = { + connection_id: z.string().optional(), + number: z.number(), + title: z.string(), + url: z + .string() + .url() + .refine((url) => new URL(url).protocol === "https:"), + author: z.string(), + agent: z.boolean(), + merged_at: z.string(), + merge_hours: z.number().nullable(), + repo: z.string(), + source_repo: z.string(), + source_branch: z.string(), + branch_cost: z.object(branchCostFields), +}; +const snapshotFields = { + source_provider: z.enum(["github", "gitlab", "mixed"]), + repos: z.array(z.string()), + unmatched_logins: z.array(z.string()), + unlinked_branches: z.array(z.object(branchSpendFields)), + captured_at: z.string(), + periods: periods(periodSchema), + people: z.array(z.object(personFields)), + pulls: periods(z.array(z.object(pullFields))), +}; +export const observedSnapshotSchema = z.object(snapshotFields); +const settingsFields = { + id: z.string().optional(), + source_provider: z.enum(["github", "gitlab"]), + api_url: z.string(), + repos: z.array(z.string()), + has_token: z.boolean(), + connection_type: z.enum(["token", "app"]), + update_interval_minutes: z.number(), + ready: z.boolean(), +}; +export const observedConnectionSchema = z.object(settingsFields); +export const observedSettingsSchema = z.object({ + ...settingsFields, + connections: z.array(observedConnectionSchema).optional(), +}); +export type ObservedConnection = z.infer; +const statusFields = { + running: z.boolean(), + phase: z.string(), + stage: z.string(), + done: z.number(), + total: z.number(), + error: z.string().nullable(), + finished_at: z.string().nullable().optional(), +}; +export const observedStatusSchema = z.object(statusFields); +export const observedReportResponseSchema = z.object({ report: observedSnapshotSchema.nullable() }); +export type ObservedSettings = z.infer; +export type ObservedStatus = z.infer; + +export type ObservedSnapshot = z.infer; +export type ObservedPerson = ObservedSnapshot["people"][number]; +export type ObservedPull = ObservedSnapshot["pulls"]["current"][number]; +export type Period = keyof ObservedSnapshot["periods"]; +export type Comparison = Exclude; +export type PeopleSort = "merged" | "spend" | "cost" | "name"; + +export const number = (value: number | null) => + value === null ? "Unavailable" : value.toLocaleString("en-US", { maximumFractionDigits: 1 }); +export function money(value: number | null) { + if (value === null) return "Unavailable"; + if (value > 0 && value < 0.01) return "<$0.01"; + return value.toLocaleString("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 2 }); +} +export function duration(value: number | null) { + if (value === null) return "Unavailable"; + if (value === 0) return "0m"; + if (value < 1 / 60) return "<1m"; + if (value < 1) return `${number(value * 60)}m`; + return `${number(value)}h`; +} +export const change = (current: number, baseline: number) => + baseline === 0 ? null : ((current - baseline) / baseline) * 100; + +export const accountLogins = (value: string) => [ + ...new Set( + value + .split(/[\s,]+/) + .map((login) => login.replace(/^@/, "").toLowerCase()) + .filter(Boolean), + ), +]; +export const repositoryNames = (value: string) => [ + ...new Set( + value + .split(/[\s,]+/) + .filter(Boolean) + .map((repo) => { + const path = repo.replace(/^https:\/\/[^/]+\//, ""); + return path.replace(/\/$/, "").replace(/\.git$/, ""); + }), + ), +]; + +export function dateRange(window: { start: string; end: string }) { + const date = (value: string) => + new Date(`${value}T00:00:00Z`).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + timeZone: "UTC", + }); + return `${date(window.start)} – ${date(window.end)}, ${window.end.slice(0, 4)}`; +} + +export function visiblePeople(people: ObservedPerson[], query: string, sort: PeopleSort) { + const value = (person: ObservedPerson) => { + const current = person.periods.current; + if (sort === "spend") return current.spend_observation === "no_records" ? -1 : current.gateway_recorded_spend; + if (sort === "cost") return current.recorded_spend_per_attributed_pr ?? -1; + return current.merged_prs; + }; + return people + .filter((person) => + [person.name, person.email, ...person.logins].join(" ").toLowerCase().includes(query.trim().toLowerCase()), + ) + .toSorted((a, b) => (sort === "name" ? a.name.localeCompare(b.name) : value(b) - value(a))); +} + +export function weeklyMerges(snapshot: ObservedSnapshot, period: Period) { + const start = Date.parse(`${snapshot.periods[period].window.start}T00:00:00Z`); + const end = Date.parse(`${snapshot.periods[period].window.end}T00:00:00Z`) + 86_400_000; + const days = (end - start) / 86_400_000; + return Array.from( + { length: Math.ceil(days / 7) }, + (_, week) => + snapshot.pulls[period].filter((pull) => { + const day = (Date.parse(pull.merged_at) - start) / 86_400_000; + return day >= week * 7 && day < Math.min((week + 1) * 7, days); + }).length, + ); +} + +export function syncMessage(status: ObservedStatus, report: ObservedSnapshot | null) { + if (status.running) return status.total ? `${status.stage} · ${status.done} / ${status.total}` : status.stage; + return report ? `Updated ${new Date(report.captured_at).toLocaleString()}` : ""; +} + +export function recordedBranches(snapshot: ObservedSnapshot) { + const matched = snapshot.pulls.current.flatMap((pull) => { + const cost = pull.branch_cost; + if (cost.status !== "matched" || cost.spend === null) return []; + return [{ repo: cost.repo, branch: cost.branch, spend: cost.spend, requests: cost.requests }]; + }); + return [ + ...new Map([...matched, ...snapshot.unlinked_branches].map((row) => [`${row.repo}\n${row.branch}`, row])).values(), + ].toSorted((a, b) => b.spend - a.spend); +} + +export function changeTerms(provider: ObservedSnapshot["source_provider"]) { + if (provider === "mixed") + return { singular: "change", plural: "changes", requests: "Merged changes", lower: "merged changes" }; + return provider === "gitlab" + ? { singular: "MR", plural: "MRs", requests: "Merge requests", lower: "merge requests" } + : { singular: "PR", plural: "PRs", requests: "Pull requests", lower: "pull requests" }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.test.ts new file mode 100644 index 00000000000..23ad2dc85b6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { observedSnapshotSchema, weeklyMerges, type Period } from "./observedData"; +import { createObservedDemo } from "./observedDemo"; + +describe("observed sample report", () => { + it("clamps the year-ago comparison on leap day while keeping the selected length", () => { + const report = createObservedDemo(28, new Date("2024-03-01T12:00:00Z")); + expect(report.periods.current.window.end).toBe("2024-02-29"); + expect(report.periods.last_year.window).toEqual({ start: "2023-02-01", end: "2023-02-28" }); + }); + + it.each([7, 28, 90])("keeps totals, attribution, costs, and comparison windows consistent for %i days", (days) => { + const report = createObservedDemo(days, new Date("2026-10-03T14:00:00Z")); + expect(observedSnapshotSchema.safeParse(report).success).toBe(true); + for (const period of ["current", "previous", "last_year"] satisfies Period[]) { + const metrics = report.periods[period]; + const pulls = report.pulls[period]; + const people = report.people.map((person) => person.periods[period]); + const start = Date.parse(metrics.window.start); + const end = Date.parse(metrics.window.end) + 86_400_000; + expect((end - start) / 86_400_000).toBe(days); + expect(metrics.merged_prs).toBe(pulls.length); + expect(new Set(pulls.map((pull) => pull.url)).size).toBe(pulls.length); + expect(weeklyMerges(report, period).reduce((sum, value) => sum + value, 0)).toBe(pulls.length); + expect(pulls.every((pull) => Date.parse(pull.merged_at) >= start && Date.parse(pull.merged_at) < end)).toBe(true); + expect(metrics.matched_users_recorded_spend).toBe( + people.reduce((sum, person) => sum + person.gateway_recorded_spend, 0), + ); + expect(people.reduce((sum, person) => sum + person.merged_prs, 0)).toBe(metrics.matched_internal_prs); + for (const person of people) { + const attributed = pulls.filter((pull) => person.pr_urls.includes(pull.url)); + expect(attributed).toHaveLength(person.merged_prs); + expect(person.direct_authored).toBe(attributed.filter((pull) => !pull.agent).length); + expect(person.declared_agent_owned).toBe(attributed.filter((pull) => pull.agent).length); + expect(person.recorded_spend_per_attributed_pr).toBe(person.gateway_recorded_spend / person.merged_prs); + } + } + expect(Date.parse(report.periods.previous.window.end) + 86_400_000).toBe( + Date.parse(report.periods.current.window.start), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.ts b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.ts new file mode 100644 index 00000000000..fb43fa8df3f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/observedDemo.ts @@ -0,0 +1,120 @@ +import type { ObservedPerson, ObservedPull, ObservedSnapshot, Period } from "./observedData"; + +const DAY = 86_400_000; +const engineers = [ + { name: "Alex Rivera", login: "alex-demo", weekly: [8, 6, 4] }, + { name: "Sam Chen", login: "sam-demo", weekly: [6, 5, 4] }, + { name: "Jordan Lee", login: "jordan-demo", weekly: [4, 4, 3] }, +]; +const titles = [ + "Add repository search", + "Fix retry handling", + "Speed up activity queries", + "Add usage export", + "Improve connection setup", + "Fix pagination", +]; +const isoDate = (timestamp: number) => new Date(timestamp).toISOString().slice(0, 10); + +function median(pulls: ObservedPull[]) { + const hours = pulls.map((pull) => pull.merge_hours ?? 0).toSorted((a, b) => a - b); + const middle = Math.floor(hours.length / 2); + if (!hours.length) return null; + return hours.length % 2 ? hours[middle] : (hours[middle - 1] + hours[middle]) / 2; +} + +function samplePeriod(days: number, end: number, comparison: number) { + const start = end - (days - 1) * DAY; + const people = engineers.map((engineer, index) => { + const count = Math.round((engineer.weekly[comparison] * days) / 7); + const pulls: ObservedPull[] = Array.from({ length: count }, (_, position) => { + const number = 10000 * (comparison + 1) + index * 1000 + position; + const repo = index === 1 ? "demo/api" : "demo/web"; + const branch = `feature/sample-${number}`; + const agent = position % 5 === 0; + return { + connection_id: index === 1 ? "demo-gitlab" : "demo-github", + number, + title: titles[position % titles.length], + url: `https://example.com/${repo}/changes/${number}`, + author: agent ? "demo-agent" : engineer.login, + agent, + merged_at: new Date(start + Math.floor((position * days) / count) * DAY + 12 * 3_600_000).toISOString(), + merge_hours: position === 0 ? 16 / 3600 : 4 + ((position * 7) % 24) + comparison * 6, + repo, + source_repo: repo, + source_branch: branch, + branch_cost: { repo, branch, spend: 2 + (position % 4), requests: 20 + position, status: "matched" }, + }; + }); + const spend = pulls.reduce((total, pull) => total + (pull.branch_cost.spend ?? 0), 0); + const metrics: ObservedPerson["periods"]["current"] = { + merged_prs: pulls.length, + prs_per_week: (pulls.length * 7) / days, + median_merge_hours: median(pulls), + direct_authored: pulls.filter((pull) => !pull.agent).length, + declared_agent_owned: pulls.filter((pull) => pull.agent).length, + gateway_recorded_spend: spend, + recorded_spend_per_attributed_pr: pulls.length ? spend / pulls.length : null, + spend_observation: "records_present", + pr_urls: pulls.map((pull) => pull.url), + }; + return { metrics, pulls }; + }); + const pulls = people.flatMap((person) => person.pulls).toSorted((a, b) => b.merged_at.localeCompare(a.merged_at)); + const metrics: ObservedSnapshot["periods"]["current"] = { + window: { start: isoDate(start), end: isoDate(end) }, + merged_prs: pulls.length, + median_merge_hours: median(pulls), + human_authored: pulls.filter((pull) => !pull.agent).length, + agent_authored: pulls.filter((pull) => pull.agent).length, + missing_author: 0, + agents_without_requester: 0, + matched_internal_prs: pulls.length, + new_bug_labeled_issues: Math.round(((comparison + 1) * days) / 7), + new_regression_labeled_issues: comparison, + explicitly_titled_revert_prs: 0, + matched_users_recorded_spend: people.reduce((total, person) => total + person.metrics.gateway_recorded_spend, 0), + spend_observation: "records_present", + human_summary: { median_merge_hours: median(pulls.filter((pull) => !pull.agent)) }, + }; + return { metrics, pulls, people }; +} + +export function createObservedDemo(days: number, now = new Date()): ObservedSnapshot { + const end = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - DAY; + const yearAgo = new Date(end); + const lastYear = yearAgo.getUTCFullYear() - 1; + const month = yearAgo.getUTCMonth(); + const lastDay = new Date(Date.UTC(lastYear, month + 1, 0)).getUTCDate(); + const lastYearEnd = Date.UTC(lastYear, month, Math.min(yearAgo.getUTCDate(), lastDay)); + const periods = { + current: samplePeriod(days, end, 0), + previous: samplePeriod(days, end - days * DAY, 1), + last_year: samplePeriod(days, lastYearEnd, 2), + }; + const personPeriod = (period: Period, index: number) => periods[period].people[index].metrics; + return { + source_provider: "mixed", + repos: ["demo/web", "demo/api"], + unmatched_logins: [], + unlinked_branches: [], + captured_at: now.toISOString(), + periods: { + current: periods.current.metrics, + previous: periods.previous.metrics, + last_year: periods.last_year.metrics, + }, + people: engineers.map((engineer, index) => ({ + name: engineer.name, + email: `${engineer.login}@example.com`, + logins: [engineer.login], + periods: { + current: personPeriod("current", index), + previous: personPeriod("previous", index), + last_year: personPeriod("last_year", index), + }, + })), + pulls: { current: periods.current.pulls, previous: periods.previous.pulls, last_year: periods.last_year.pulls }, + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/useObservedReport.ts b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/useObservedReport.ts new file mode 100644 index 00000000000..989946e2a27 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/useObservedReport.ts @@ -0,0 +1,60 @@ +import { useCallback, useEffect, useState } from "react"; +import { apiClient } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { + observedSettingsSchema, + observedReportResponseSchema, + observedStatusSchema, + type ObservedSettings, + type ObservedSnapshot, + type ObservedStatus, +} from "./observedData"; + +export type ObservedViewData = { settings: ObservedSettings; report: ObservedSnapshot | null; status: ObservedStatus }; + +export function useObservedReport(accessToken: string) { + const [data, setData] = useState(null); + const [error, setError] = useState(""); + const [revision, setRevision] = useState(0); + const refresh = useCallback(() => setRevision((value) => value + 1), []); + useEffect(() => { + const controller = new AbortController(); + let timer: ReturnType; + const options = { accessToken, signal: controller.signal }; + async function poll(previous?: ObservedViewData) { + try { + const status = observedStatusSchema.parse( + await apiClient.get("/roi-calculator/observed/sync", options), + ); + const finished = previous?.status.running && !status.running; + const changed = !previous || previous.status.finished_at !== status.finished_at || finished; + const updated = changed + ? await Promise.all([ + apiClient + .get("/roi-calculator/observed/settings", options) + .then((value) => observedSettingsSchema.parse(value)), + apiClient + .get("/roi-calculator/observed/report", options) + .then((value) => observedReportResponseSchema.parse(value)), + ]) + : null; + if (controller.signal.aborted) return; + const existing = previous ? { ...previous, status } : null; + const next = updated ? { settings: updated[0], report: updated[1].report, status } : existing; + setData(next); + setError(""); + timer = setTimeout(() => void poll(next ?? undefined), status.running ? 2000 : 30000); + } catch (reason) { + if (controller.signal.aborted) return; + setError(extractProxyErrorMessage(reason)); + timer = setTimeout(() => void poll(previous), 10000); + } + } + void poll(); + return () => { + controller.abort(); + clearTimeout(timer); + }; + }, [accessToken, revision]); + return { data, error, refresh }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/page.tsx index 329ecbc0fe6..da1688e342f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/page.tsx @@ -1,9 +1,10 @@ "use client"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import ROICalculatorView from "./_components/ROICalculatorView"; +import ObservedROIView from "./_components/ObservedROIView"; export default function ROICalculatorPage() { - const { accessToken, userRole, isViewOnly } = useAuthorized(); - return ; + const { accessToken, isViewOnly } = useAuthorized(); + if (!accessToken) return null; + return ; } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0dde64137a5..5478772bc29 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -14331,6 +14331,129 @@ export interface paths { patch?: never; trace?: never; }; + "/roi-calculator/observed/apps": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Observed Apps */ + get: operations["observed_apps_roi_calculator_observed_apps_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/identities": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Observed Identities */ + get: operations["get_observed_identities_roi_calculator_observed_identities_get"]; + /** Save Observed Identities */ + put: operations["save_observed_identities_roi_calculator_observed_identities_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/oauth/{provider}/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Start Observed Authorization */ + post: operations["start_observed_authorization_roi_calculator_observed_oauth__provider__start_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Observed Report */ + get: operations["get_observed_report_roi_calculator_observed_report_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/repositories": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Observed Repositories */ + get: operations["observed_repositories_roi_calculator_observed_repositories_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/settings": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Observed Settings */ + get: operations["get_observed_settings_roi_calculator_observed_settings_get"]; + /** Save Observed Settings */ + put: operations["save_observed_settings_roi_calculator_observed_settings_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/roi-calculator/observed/sync": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Observed Sync */ + get: operations["get_observed_sync_roi_calculator_observed_sync_get"]; + put?: never; + /** Start Observed Sync */ + post: operations["start_observed_sync_roi_calculator_observed_sync_post"]; + /** Cancel Observed Sync */ + delete: operations["cancel_observed_sync_roi_calculator_observed_sync_delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/roi-calculator/report": { parameters: { query?: never; @@ -39236,6 +39359,373 @@ export interface components { [key: string]: unknown; } | null; }; + /** ObservedAccount */ + ObservedAccount: { + /** Connection Id */ + connection_id: string; + /** Login */ + login: string; + }; + /** ObservedApp */ + ObservedApp: { + /** Api Url */ + api_url?: string | null; + /** Callback Url */ + callback_url?: string | null; + /** + * Can Install + * @default false + */ + can_install: boolean; + /** Configured */ + configured: boolean; + }; + /** ObservedApps */ + ObservedApps: { + github: components["schemas"]["ObservedApp"]; + gitlab: components["schemas"]["ObservedApp"]; + }; + /** ObservedAuthorization */ + ObservedAuthorization: { + /** Url */ + url: string; + }; + /** ObservedConnection */ + ObservedConnection: { + /** Api Url */ + api_url: string; + /** + * Connection Type + * @enum {string} + */ + connection_type: "token" | "app"; + /** Has Token */ + has_token: boolean; + /** + * Id + * @default + */ + id: string; + /** Ready */ + ready: boolean; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab"; + /** Update Interval Minutes */ + update_interval_minutes: number; + }; + /** ObservedConnectionIdentities */ + ObservedConnectionIdentities: { + /** Api Url */ + api_url: string; + /** Id */ + id: string; + /** Identity Map */ + identity_map: { + [key: string]: string; + }; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab"; + /** Unmatched Logins */ + unmatched_logins: string[]; + }; + /** ObservedHumanSummary */ + ObservedHumanSummary: { + /** Median Merge Hours */ + median_merge_hours: number | null; + }; + /** ObservedIdentities */ + ObservedIdentities: { + /** + * Connections + * @default [] + */ + connections: components["schemas"]["ObservedConnectionIdentities"][]; + /** Gateway Emails */ + gateway_emails: string[]; + /** Identity Map */ + identity_map: { + [key: string]: string; + }; + /** Unmatched Logins */ + unmatched_logins: string[]; + }; + /** ObservedIdentityUpdate */ + ObservedIdentityUpdate: { + /** Accounts */ + accounts?: components["schemas"]["ObservedAccount"][] | null; + /** Email */ + email: string; + /** + * Logins + * @default [] + */ + logins: string[]; + }; + /** ObservedPeriod */ + ObservedPeriod: { + /** Agent Authored */ + agent_authored: number; + /** Agents Without Requester */ + agents_without_requester: number; + /** Explicitly Titled Revert Prs */ + explicitly_titled_revert_prs: number; + /** Human Authored */ + human_authored: number; + human_summary: components["schemas"]["ObservedHumanSummary"]; + /** Matched Internal Prs */ + matched_internal_prs: number; + /** Matched Users Recorded Spend */ + matched_users_recorded_spend: number; + /** Median Merge Hours */ + median_merge_hours: number | null; + /** Merged Prs */ + merged_prs: number; + /** Missing Author */ + missing_author: number; + /** New Bug Labeled Issues */ + new_bug_labeled_issues: number | null; + /** New Regression Labeled Issues */ + new_regression_labeled_issues: number | null; + /** + * Spend Observation + * @enum {string} + */ + spend_observation: "records_present" | "no_records"; + window: components["schemas"]["ObservedWindow"]; + }; + /** ObservedPeriods */ + ObservedPeriods: { + current: components["schemas"]["ObservedPeriod"]; + last_year: components["schemas"]["ObservedPeriod"]; + previous: components["schemas"]["ObservedPeriod"]; + }; + /** ObservedPerson */ + ObservedPerson: { + /** + * Accounts + * @default [] + */ + accounts: components["schemas"]["ObservedAccount"][]; + /** Email */ + email: string; + /** Logins */ + logins: string[]; + /** Name */ + name: string; + periods: components["schemas"]["ObservedPersonPeriods"]; + }; + /** ObservedPersonPeriod */ + ObservedPersonPeriod: { + /** Declared Agent Owned */ + declared_agent_owned: number; + /** Direct Authored */ + direct_authored: number; + /** Gateway Recorded Spend */ + gateway_recorded_spend: number; + /** Median Merge Hours */ + median_merge_hours: number | null; + /** Merged Prs */ + merged_prs: number; + /** Pr Urls */ + pr_urls: string[]; + /** Prs Per Week */ + prs_per_week: number; + /** Recorded Spend Per Attributed Pr */ + recorded_spend_per_attributed_pr: number | null; + /** + * Spend Observation + * @enum {string} + */ + spend_observation: "records_present" | "no_records"; + }; + /** ObservedPersonPeriods */ + ObservedPersonPeriods: { + current: components["schemas"]["ObservedPersonPeriod"]; + last_year: components["schemas"]["ObservedPersonPeriod"]; + previous: components["schemas"]["ObservedPersonPeriod"]; + }; + /** ObservedPullPeriods */ + ObservedPullPeriods: { + /** Current */ + current: components["schemas"]["ObservedPullResponse"][]; + /** Last Year */ + last_year: components["schemas"]["ObservedPullResponse"][]; + /** Previous */ + previous: components["schemas"]["ObservedPullResponse"][]; + }; + /** ObservedPullResponse */ + ObservedPullResponse: { + /** + * Agent + * @default false + */ + agent: boolean; + /** Author */ + author: string; + branch_cost: components["schemas"]["ROIBranchAttribution"]; + /** + * Connection Id + * @default + */ + connection_id: string; + /** Created At */ + created_at?: string | null; + /** Merge Hours */ + merge_hours: number | null; + /** + * Merged At + * Format: date-time + */ + merged_at: string; + /** Number */ + number: number; + /** + * Profile Email + * @default + */ + profile_email: string; + /** Repo */ + repo: string; + /** + * Requester + * @default + */ + requester: string; + /** + * Source Branch + * @default + */ + source_branch: string; + /** + * Source Repo + * @default + */ + source_repo: string; + /** Title */ + title: string; + /** Url */ + url: string; + }; + /** ObservedReport */ + ObservedReport: { + /** + * Captured At + * Format: date-time + */ + captured_at: string; + /** + * Connections + * @default [] + */ + connections: components["schemas"]["ObservedSource"][]; + /** People */ + people: components["schemas"]["ObservedPerson"][]; + periods: components["schemas"]["ObservedPeriods"]; + pulls: components["schemas"]["ObservedPullPeriods"]; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab" | "mixed"; + /** Unlinked Branches */ + unlinked_branches: components["schemas"]["ROIBranchSpend"][]; + /** Unmatched Logins */ + unmatched_logins: string[]; + }; + /** ObservedReportResponse */ + ObservedReportResponse: { + report: components["schemas"]["ObservedReport"] | null; + }; + /** ObservedSettings */ + ObservedSettings: { + /** Api Url */ + api_url: string; + /** + * Connection Type + * @enum {string} + */ + connection_type: "token" | "app"; + /** + * Connections + * @default [] + */ + connections: components["schemas"]["ObservedConnection"][]; + /** Has Token */ + has_token: boolean; + /** + * Id + * @default + */ + id: string; + /** Ready */ + ready: boolean; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab"; + /** Update Interval Minutes */ + update_interval_minutes: number; + }; + /** ObservedSettingsUpdate */ + ObservedSettingsUpdate: { + /** Api Url */ + api_url: string; + /** Connection Id */ + connection_id?: string | null; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab"; + /** Token */ + token?: string | null; + /** Update Interval Minutes */ + update_interval_minutes?: number | null; + }; + /** ObservedSource */ + ObservedSource: { + /** Api Url */ + api_url: string; + /** Id */ + id: string; + /** Repos */ + repos: string[]; + /** + * Source Provider + * @enum {string} + */ + source_provider: "github" | "gitlab"; + }; + /** ObservedWindow */ + ObservedWindow: { + /** + * End + * Format: date + */ + end: string; + /** + * Start + * Format: date + */ + start: string; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect. @@ -41591,6 +42081,12 @@ export interface components { }; /** Ready */ ready: boolean; + /** + * Report Mode + * @default legacy + * @enum {string} + */ + report_mode: "legacy" | "observed"; /** Repos */ repos: string[]; /** @@ -41620,6 +42116,8 @@ export interface components { gitlab_api_url?: string | null; /** Gitlab Token */ gitlab_token?: string | null; + /** Report Mode */ + report_mode?: ("legacy" | "observed") | null; /** Repos */ repos?: string[] | null; /** Source Provider */ @@ -69452,6 +69950,289 @@ export interface operations { }; }; }; + observed_apps_roi_calculator_observed_apps_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedApps"]; + }; + }; + }; + }; + get_observed_identities_roi_calculator_observed_identities_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedIdentities"]; + }; + }; + }; + }; + save_observed_identities_roi_calculator_observed_identities_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ObservedIdentityUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedReportResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + start_observed_authorization_roi_calculator_observed_oauth__provider__start_post: { + parameters: { + query?: { + install?: boolean; + }; + header?: never; + path: { + provider: "github" | "gitlab"; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedAuthorization"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_observed_report_roi_calculator_observed_report_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedReportResponse"]; + }; + }; + }; + }; + observed_repositories_roi_calculator_observed_repositories_get: { + parameters: { + query?: { + connection?: string | null; + query?: string; + page?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ROIRepositoriesResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_observed_settings_roi_calculator_observed_settings_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedSettings"]; + }; + }; + }; + }; + save_observed_settings_roi_calculator_observed_settings_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ObservedSettingsUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ObservedSettings"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_observed_sync_roi_calculator_observed_sync_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ROISyncStatus"]; + }; + }; + }; + }; + start_observed_sync_roi_calculator_observed_sync_post: { + parameters: { + query?: { + days?: number | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ROISyncStatus"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + cancel_observed_sync_roi_calculator_observed_sync_delete: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ROISyncStatus"]; + }; + }; + }; + }; get_roi_calculator_report_roi_calculator_report_get: { parameters: { query?: {