fix(usage): count a subscription fee only for accounts behind a current deployment

Usage listed a fee row for an account no deployment uses while another ChatGPT
deployment existed and dropped it when none did. It now lists only accounts behind a
current deployment in both cases. Adds a real proxy and Postgres integration test for
the fee API and for ChatGPT Responses rows on a deployment's own login, and names the
card's multi-component test with the integration suffix.
This commit is contained in:
mateo-berri 2026-10-29 18:18:56 -07:00
parent 572cb1faa3
commit 32fe58ad00
4 changed files with 308 additions and 7 deletions

View file

@ -169,7 +169,7 @@ def build_subscription_usage(
deployment_tuple: Final = tuple(deployments)
fees_by_account: Final = {(row.custom_llm_provider, row.account_id): row for row in fee_rows}
deployment_keys: Final = {(d.custom_llm_provider, d.account_id) for d in deployment_tuple}
account_keys: Final = sorted(deployment_keys | set(fees_by_account), key=lambda key: (key[0], key[1] or "~"))
account_keys: Final = sorted(deployment_keys, key=lambda key: (key[0], key[1] or "~"))
accounts: Final = tuple(
_account_usage(
provider,

View file

@ -0,0 +1,302 @@
import uuid
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Final
import httpx
import pytest
from integration._support import codex_vendor as cv
from integration._support import responses_vendor as rv
from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, string_value
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Wire, wire_server
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
_MODEL: Final = "chatgpt/gpt-5.5"
_PROVIDER: Final = "chatgpt"
_FEE_TABLE_QUERY: Final = (
'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE custom_llm_provider = %s'
" AND account_id = %s"
)
_FEE_ROW_QUERY: Final = (
'SELECT subscription_account_id FROM "LiteLLM_SubscriptionAccountTable" WHERE subscription_account_id = %s'
)
_MONTHLY_FEE: Final = 200.0
_BILLING_PERIOD_START: Final = "2026-01-15"
_RANGE: Final = {"start_date": "2026-03-01", "end_date": "2026-03-31"}
_PERIOD_IN_RANGE: Final = ("2026-03-15", "2026-04-14")
_TOKENS: Final = TypeAdapter(int)
class _Metrics(BaseModel):
model_config = ConfigDict(frozen=True)
spend: float
prompt_tokens: int
completion_tokens: int
api_requests: int
successful_requests: int
failed_requests: int
class _ProviderUsage(BaseModel):
model_config = ConfigDict(frozen=True)
metrics: _Metrics
metadata: Mapping[str, JsonValue]
class _Breakdown(BaseModel):
model_config = ConfigDict(frozen=True)
providers: Mapping[str, _ProviderUsage]
class _Day(BaseModel):
model_config = ConfigDict(frozen=True)
breakdown: _Breakdown
class _DailyActivity(BaseModel):
model_config = ConfigDict(frozen=True)
results: tuple[_Day, ...]
class _Fee(BaseModel):
model_config = ConfigDict(frozen=True)
subscription_account_id: str
custom_llm_provider: str
account_id: str
monthly_fee: float
currency: str
billing_period_start: str
class _Period(BaseModel):
model_config = ConfigDict(frozen=True)
start: str
end: str
class _Account(BaseModel):
model_config = ConfigDict(frozen=True)
custom_llm_provider: str
account_id: str | None
deployments: tuple[str, ...]
fee: _Fee | None
billing_periods: tuple[_Period, ...]
fixed_cost: float | None
class _SubscriptionUsage(BaseModel):
model_config = ConfigDict(frozen=True)
accounts: tuple[_Account, ...]
@dataclass(frozen=True, slots=True)
class _Rig:
wire: Wire
proxy: OwnedProxy
model: str
@property
def gateway(self) -> Gateway:
return self.proxy.gateway
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("chatgpt-subscription-usage")
no_default_login: Final = directory / "no-default-login"
no_default_login.mkdir()
deployment_login: Final = cv.login(directory) / "auth.json"
with gateway_from_environment() as gateway, wire_server(cv.CodexVendor().respond) as wire:
overrides: Final = {"CHATGPT_TOKEN_DIR": str(no_default_login), "CHATGPT_API_BASE": wire.url}
config: Final = cv.proxy_config(directory, probe=False)
with (
owned_proxy_process(gateway, directory, overrides, config=config, workers=2) as owned,
owned.gateway.scenario() as scenario,
):
model: Final = scenario.model(
model=_MODEL, api_base=wire.url, api_key=None, chatgpt_auth_file=str(deployment_login)
)
yield _Rig(wire, owned, model)
def _responses_body(model: str, marker: str, *, stream: bool) -> Mapping[str, JsonValue]:
return {
"model": model,
"input": [{"role": "user", "content": f"Say marker-{marker}"}],
"stream": stream,
"cache": {"no-cache": True},
}
def _assert_answered(response: httpx.Response, marker: str, content_type: str) -> None:
assert response.status_code == 200, response.text
assert response.headers["content-type"].startswith(content_type), response.headers
assert rv.answer(marker) in response.text, response.text
def _chatgpt_usage_on(gateway: Gateway, day: str) -> _ProviderUsage | None:
response: Final = gateway.request("GET", "/user/daily/activity", params={"start_date": day, "end_date": day})
assert response.status_code == 200, response.text
activity: Final = _DailyActivity.model_validate_json(response.content)
buckets: Final = tuple(
result.breakdown.providers[_PROVIDER] for result in activity.results if _PROVIDER in result.breakdown.providers
)
assert len(buckets) <= 1, response.text
return buckets[0] if buckets else None
def _usage(gateway: Gateway, *, key: str | None = None) -> httpx.Response:
return gateway.request("GET", "/subscription_accounts/usage", params=_RANGE, key=key)
def _accounts(gateway: Gateway) -> tuple[_Account, ...]:
response: Final = _usage(gateway)
assert response.status_code == 200, response.text
return _SubscriptionUsage.model_validate_json(response.content).accounts
def _deployment_account(rig: _Rig) -> _Account:
(account,) = _accounts(rig.gateway)
assert (account.custom_llm_provider, account.account_id, account.deployments) == (
_PROVIDER,
cv.ACCOUNT,
(rig.model,),
), account
return account
def _assert_no_fee(account: _Account) -> None:
assert (account.fee, account.billing_periods, account.fixed_cost) == (None, (), None), account
def _fee_body(account_id: str, monthly_fee: float = _MONTHLY_FEE) -> Mapping[str, JsonValue]:
return {
"custom_llm_provider": _PROVIDER,
"account_id": account_id,
"monthly_fee": monthly_fee,
"currency": "USD",
"billing_period_start": _BILLING_PERIOD_START,
"label": "integration",
}
def _delete_fee_if_present(gateway: Gateway, identity: str) -> None:
if gateway.rows(_FEE_ROW_QUERY, (identity,)):
gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity})
assert gateway.rows(_FEE_ROW_QUERY, (identity,)) == []
def _set_fee(scenario: Scenario, account_id: str) -> str:
created: Final = scenario.gateway.post("/subscription_accounts/new", _fee_body(account_id))
identity: Final = string_value(created["subscription_account_id"])
scenario.cleanups.callback(_delete_fee_if_present, scenario.gateway, identity)
return identity
def _assert_fee_reported(account: _Account, identity: str) -> None:
assert account.fee is not None, account
assert (
account.fee.subscription_account_id,
account.fee.account_id,
account.fee.monthly_fee,
account.fee.currency,
account.fee.billing_period_start,
) == (identity, cv.ACCOUNT, _MONTHLY_FEE, "USD", _BILLING_PERIOD_START), account
assert tuple((period.start, period.end) for period in account.billing_periods) == (_PERIOD_IN_RANGE,), account
assert account.fixed_cost == _MONTHLY_FEE, account
def test_responses_calls_on_a_deployment_login_count_as_subscription_covered_chatgpt_usage(rig: _Rig) -> None:
day: Final = datetime.now(UTC).date().isoformat()
rig.wire.drain()
plain_marker: Final = uuid.uuid4().hex
stream_marker: Final = uuid.uuid4().hex
plain: Final = rig.gateway.request("POST", "/v1/responses", _responses_body(rig.model, plain_marker, stream=False))
_assert_answered(plain, plain_marker, "application/json")
streamed: Final = rig.gateway.request(
"POST", "/v1/responses", _responses_body(rig.model, stream_marker, stream=True)
)
_assert_answered(streamed, stream_marker, "text/event-stream")
forwarded: Final = rig.wire.drain()
assert len(forwarded) == 2, [request.target for request in forwarded]
for request, marker in zip(forwarded, (plain_marker, stream_marker), strict=True):
cv.forwarded(request, marker)
usage: Final = eventually(
lambda: _chatgpt_usage_on(rig.gateway, day),
lambda found: found is not None and found.metrics.api_requests >= 2,
seconds=70,
)
assert usage is not None
assert (
usage.metrics.api_requests,
usage.metrics.successful_requests,
usage.metrics.failed_requests,
usage.metrics.spend,
) == (2, 2, 0, 0), usage
assert (usage.metrics.prompt_tokens, usage.metrics.completion_tokens) == (
2 * _TOKENS.validate_python(cv.USAGE["input_tokens"]),
2 * _TOKENS.validate_python(cv.USAGE["output_tokens"]),
), usage
assert usage.metadata.get("subscription_covered") is True, usage
def test_fee_on_the_deployment_account_reports_its_fixed_cost_until_deleted(rig: _Rig) -> None:
_assert_no_fee(_deployment_account(rig))
with rig.gateway.scenario() as scenario:
identity: Final = _set_fee(scenario, cv.ACCOUNT)
_assert_fee_reported(_deployment_account(rig), identity)
deleted: Final = rig.gateway.post("/subscription_accounts/delete", {"subscription_account_id": identity})
assert deleted["subscription_account_id"] == identity, deleted
assert rig.gateway.rows(_FEE_ROW_QUERY, (identity,)) == []
_assert_no_fee(_deployment_account(rig))
def test_second_fee_for_the_same_account_answers_400_and_keeps_the_first(rig: _Rig) -> None:
with rig.gateway.scenario() as scenario:
identity: Final = _set_fee(scenario, cv.ACCOUNT)
duplicate: Final = rig.gateway.request(
"POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT, monthly_fee=3 * _MONTHLY_FEE)
)
assert duplicate.status_code == 400, duplicate.text
assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == [{"subscription_account_id": identity}]
_assert_fee_reported(_deployment_account(rig), identity)
def test_update_and_delete_of_an_unknown_fee_answer_404(rig: _Rig) -> None:
unknown: Final = str(uuid.uuid4())
updated: Final = rig.gateway.request(
"POST", "/subscription_accounts/update", {"subscription_account_id": unknown, "monthly_fee": _MONTHLY_FEE}
)
assert updated.status_code == 404, updated.text
deleted: Final = rig.gateway.request("POST", "/subscription_accounts/delete", {"subscription_account_id": unknown})
assert deleted.status_code == 404, deleted.text
assert rig.gateway.rows(_FEE_ROW_QUERY, (unknown,)) == []
def test_internal_user_is_refused_setting_a_fee_and_reading_usage(rig: _Rig) -> None:
with rig.gateway.scenario() as scenario:
key: Final = scenario.key(user_id=scenario.user(user_role="internal_user"))
created: Final = rig.gateway.request("POST", "/subscription_accounts/new", _fee_body(cv.ACCOUNT), key=key)
assert created.status_code == 401, created.text
assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, cv.ACCOUNT)) == []
usage: Final = _usage(rig.gateway, key=key)
assert usage.status_code == 401, usage.text
def test_fee_on_an_account_no_deployment_uses_is_neither_listed_nor_counted(rig: _Rig) -> None:
unused: Final = f"acct-unused-{uuid.uuid4().hex}"
with rig.gateway.scenario() as scenario:
identity: Final = _set_fee(scenario, unused)
assert rig.gateway.rows(_FEE_TABLE_QUERY, (_PROVIDER, unused)) == [{"subscription_account_id": identity}]
_assert_no_fee(_deployment_account(rig))

View file

@ -174,18 +174,17 @@ def test_fee_counts_once_per_billing_period_however_many_deployments_share_the_a
assert between_starts["accounts"][0]["billing_periods"] == []
def test_fee_without_a_matching_deployment_is_still_listed(client):
def test_fee_of_an_account_no_deployment_uses_is_neither_listed_nor_counted(client):
test_client, _ = client
other = {**_TEST_FEE, "account_id": "org-other", "monthly_fee": 200.0, "currency": "EUR"}
assert test_client.post("/subscription_accounts/new", json=other).status_code == 200
assert test_client.post("/subscription_accounts/new", json=_TEST_FEE).status_code == 200
body = _usage(test_client, "2026-10-01", "2026-10-07")
assert [account["account_id"] for account in body["accounts"]] == ["org-other", _ACCOUNT]
assert body["accounts"][0]["deployments"] == []
assert body["accounts"][0]["fee"]["currency"] == "EUR"
assert body["accounts"][0]["fixed_cost"] == 200.0
assert body["accounts"][1]["fee"] is None
(account,) = body["accounts"]
assert account["account_id"] == _ACCOUNT
assert account["fixed_cost"] == 25.0
@pytest.mark.parametrize(