From d39df0d4ac70c675657030c62bfcf1c4ee2f8be5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:58:09 -0700 Subject: [PATCH] fix(usage): keep a stored subscription fee listed after its account's deployments are removed The usage route lists every account behind a subscription deployment plus every account carrying a stored fee, and reads the fees on every proxy, so a fee stays visible, counted, and editable for past ranges after its last deployment goes. The integration test now polls every day the calls spanned, so a UTC midnight crossing no longer fails it. --- .../subscription_accounts.py | 8 +-- .../spend/test_chatgpt_subscription_usage.py | 64 ++++++++++++------- .../test_subscription_accounts.py | 30 ++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 4 files changed, 59 insertions(+), 47 deletions(-) diff --git a/litellm/proxy/management_endpoints/subscription_accounts.py b/litellm/proxy/management_endpoints/subscription_accounts.py index 4cd282ae70f..363d6cfba5d 100644 --- a/litellm/proxy/management_endpoints/subscription_accounts.py +++ b/litellm/proxy/management_endpoints/subscription_accounts.py @@ -169,7 +169,7 @@ def build_subscription_usage( deployment_tuple: Final = tuple(deployments) fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows} deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple} - account_keys: Final = sorted(deployment_keys, key=lambda key: (key[0], key[1] or "~")) + account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~")) accounts: Final = tuple( _account_usage( provider, @@ -219,8 +219,8 @@ async def get_subscription_usage( deployments: Deployments, ) -> SubscriptionUsageResponse: """ - Subscription-backed accounts behind the proxy's deployments, each with its configured - monthly fee and the fixed cost attributed to the date range: the fee once per billing + Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with + its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing period whose start date falls inside the range, however many deployments share the account. """ if not user_api_key_has_admin_view(user_api_key_dict): @@ -231,8 +231,6 @@ async def get_subscription_usage( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail={"error": "end_date must not be before start_date"} ) - if not deployments: - return build_subscription_usage((), (), start, end) return build_subscription_usage(deployments, await repository.list_all(), start, end) diff --git a/tests/integration/spend/test_chatgpt_subscription_usage.py b/tests/integration/spend/test_chatgpt_subscription_usage.py index 28018d93b9e..8f1fb2d0f74 100644 --- a/tests/integration/spend/test_chatgpt_subscription_usage.py +++ b/tests/integration/spend/test_chatgpt_subscription_usage.py @@ -145,15 +145,30 @@ def _assert_answered(response: httpx.Response, marker: str, content_type: str) - assert rv.answer(marker) in response.text, response.text -def _chatgpt_usage_on(gateway: Gateway, day: str) -> _ProviderUsage | None: - response: Final = gateway.request("GET", "/user/daily/activity", params={"start_date": day, "end_date": day}) +def _today() -> str: + return datetime.now(UTC).date().isoformat() + + +def _chatgpt_usage_between(gateway: Gateway, first_day: str, last_day: str) -> tuple[_ProviderUsage, ...]: + response: Final = gateway.request( + "GET", "/user/daily/activity", params={"start_date": first_day, "end_date": last_day} + ) assert response.status_code == 200, response.text activity: Final = _DailyActivity.model_validate_json(response.content) - buckets: Final = tuple( + return tuple( result.breakdown.providers[_PROVIDER] for result in activity.results if _PROVIDER in result.breakdown.providers ) - assert len(buckets) <= 1, response.text - return buckets[0] if buckets else None + + +def _summed(buckets: tuple[_ProviderUsage, ...]) -> _Metrics: + return _Metrics( + spend=sum(bucket.metrics.spend for bucket in buckets), + prompt_tokens=sum(bucket.metrics.prompt_tokens for bucket in buckets), + completion_tokens=sum(bucket.metrics.completion_tokens for bucket in buckets), + api_requests=sum(bucket.metrics.api_requests for bucket in buckets), + successful_requests=sum(bucket.metrics.successful_requests for bucket in buckets), + failed_requests=sum(bucket.metrics.failed_requests for bucket in buckets), + ) def _usage(gateway: Gateway, *, key: str | None = None) -> httpx.Response: @@ -218,7 +233,7 @@ def _assert_fee_reported(account: _Account, identity: str) -> None: def test_responses_calls_on_a_deployment_login_count_as_subscription_covered_chatgpt_usage(rig: _Rig) -> None: - day: Final = datetime.now(UTC).date().isoformat() + first_day: Final = _today() rig.wire.drain() plain_marker: Final = uuid.uuid4().hex stream_marker: Final = uuid.uuid4().hex @@ -232,23 +247,24 @@ def test_responses_calls_on_a_deployment_login_count_as_subscription_covered_cha assert len(forwarded) == 2, [request.target for request in forwarded] for request, marker in zip(forwarded, (plain_marker, stream_marker), strict=True): cv.forwarded(request, marker) - usage: Final = eventually( - lambda: _chatgpt_usage_on(rig.gateway, day), - lambda found: found is not None and found.metrics.api_requests >= 2, + last_day: Final = _today() + buckets: Final = eventually( + lambda: _chatgpt_usage_between(rig.gateway, first_day, last_day), + lambda found: _summed(found).api_requests >= 2, seconds=70, ) - assert usage is not None - assert ( - usage.metrics.api_requests, - usage.metrics.successful_requests, - usage.metrics.failed_requests, - usage.metrics.spend, - ) == (2, 2, 0, 0), usage - assert (usage.metrics.prompt_tokens, usage.metrics.completion_tokens) == ( + totals: Final = _summed(buckets) + assert (totals.api_requests, totals.successful_requests, totals.failed_requests, totals.spend) == ( + 2, + 2, + 0, + 0, + ), buckets + assert (totals.prompt_tokens, totals.completion_tokens) == ( 2 * _TOKENS.validate_python(cv.USAGE["input_tokens"]), 2 * _TOKENS.validate_python(cv.USAGE["output_tokens"]), - ), usage - assert usage.metadata.get("subscription_covered") is True, usage + ), buckets + assert all(bucket.metadata.get("subscription_covered") is True for bucket in buckets), buckets def test_fee_on_the_deployment_account_reports_its_fixed_cost_until_deleted(rig: _Rig) -> None: @@ -294,9 +310,13 @@ def test_internal_user_is_refused_setting_a_fee_and_reading_usage(rig: _Rig) -> assert usage.status_code == 401, usage.text -def test_fee_on_an_account_no_deployment_uses_is_neither_listed_nor_counted(rig: _Rig) -> None: +def test_fee_on_an_account_no_deployment_uses_stays_listed_and_counted(rig: _Rig) -> None: unused: Final = f"acct-unused-{uuid.uuid4().hex}" with rig.gateway.scenario() as scenario: identity: Final = _set_fee(scenario, unused) - assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, unused)) == [{"subscription_account_id": identity}] - _assert_no_fee(_deployment_account(rig)) + accounts: Final = _accounts(rig.gateway) + assert frozenset(account.account_id for account in accounts) == {cv.ACCOUNT, unused}, accounts + (unused_account,) = (account for account in accounts if account.account_id == unused) + assert unused_account.fee is not None, unused_account + assert (unused_account.deployments, unused_account.fee.subscription_account_id) == ((), identity) + assert unused_account.fixed_cost == _MONTHLY_FEE, unused_account diff --git a/tests/unit/proxy/management_endpoints/test_subscription_accounts.py b/tests/unit/proxy/management_endpoints/test_subscription_accounts.py index c50066fc803..bcd31efd4b7 100644 --- a/tests/unit/proxy/management_endpoints/test_subscription_accounts.py +++ b/tests/unit/proxy/management_endpoints/test_subscription_accounts.py @@ -174,7 +174,7 @@ def test_fee_counts_once_per_billing_period_however_many_deployments_share_the_a assert between_starts["accounts"][0]["billing_periods"] == [] -def test_fee_of_an_account_no_deployment_uses_is_neither_listed_nor_counted(client): +def test_fee_of_an_account_no_deployment_uses_stays_listed_and_counted(client): test_client, _ = client other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 200.0, "currency": "EUR"} assert test_client.post("/subscription_accounts/new", json=other).status_code == 200 @@ -182,9 +182,10 @@ def test_fee_of_an_account_no_deployment_uses_is_neither_listed_nor_counted(clie body = _usage(test_client, "2026-10-01", "2026-10-07") - (account,) = body["accounts"] - assert account["account_id"] == _ACCOUNT - assert account["fixed_cost"] == 25.0 + unused, used = body["accounts"] + assert (unused["account_id"], unused["deployments"], unused["fee"]["currency"]) == ("org-other", [], "EUR") + assert unused["fixed_cost"] == 200.0 + assert (used["account_id"], used["fixed_cost"]) == (_ACCOUNT, 25.0) @pytest.mark.parametrize( @@ -286,22 +287,15 @@ def test_rejects_an_infinite_monthly_fee(): ) -class _MissingTableStore: - async def list_all(self) -> list[LiteLLM_SubscriptionAccountTable]: - raise RuntimeError('relation "LiteLLM_SubscriptionAccountTable" does not exist') +@pytest.mark.parametrize("deployments", [()]) +def test_fee_stays_listed_after_the_last_subscription_deployment_is_removed(client): + test_client, _ = client + assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200 + body = _usage(test_client, "2026-10-01", "2026-10-07") -def test_proxy_without_subscription_deployments_answers_without_reading_fees(): - app: Final = FastAPI() - app.include_router(router) - app.dependency_overrides[get_subscription_account_repository] = _MissingTableStore - app.dependency_overrides[get_subscription_deployments] = lambda: () - app.dependency_overrides[user_api_key_auth] = _resolve_auth - - with TestClient(app) as test_client: - body = _usage(test_client, "2026-10-01", "2026-10-07") - - assert body["accounts"] == [] + (account,) = body["accounts"] + assert (account["account_id"], account["deployments"], account["fixed_cost"]) == (_ACCOUNT, [], 25.0) def test_usage_rejects_a_malformed_or_inverted_range(client): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0dc29ecf16e..9c7837e8897 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16122,8 +16122,8 @@ export interface paths { }; /** * Get Subscription Usage - * @description Subscription-backed accounts behind the proxy's deployments, each with its configured - * monthly fee and the fixed cost attributed to the date range: the fee once per billing + * @description Subscription-backed accounts behind the proxy's deployments or carrying a stored fee, each with + * its configured monthly fee and the fixed cost attributed to the date range: the fee once per billing * period whose start date falls inside the range, however many deployments share the account. */ get: operations["get_subscription_usage_subscription_accounts_usage_get"];