diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 68c5ef6b31d..af0ba9d56e6 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -54,6 +54,7 @@ - {id: mgmt.access_group.info.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "model_access_group_management_endpoints.py:600", rationale: "Access group membership query"} - {id: mgmt.mcp_server.register.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "mcp_management_endpoints.py:880", rationale: "MCP server registration"} - {id: mgmt.mcp_server.approve.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1200", rationale: "Admin approval persists"} +- {id: mgmt.budget.update.accepts_model_max_budget, module: mgmt, tier: P1, surface: api, assertions: [accepts_model_max_budget], source: "budget_management_endpoints.py:173", fail_before_fix: proven, rationale: "Per-model caps must be settable on an existing budget; model ids routinely carry dots and hyphens and the route must accept both"} - {id: mgmt.budget.update.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:155", rationale: "Limit changes apply"} - {id: mgmt.budget.delete.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "budget_management_endpoints.py:280", rationale: "Clears limits"} - {id: mgmt.budget.list.happy_path, module: mgmt, tier: P1, surface: api, assertions: [happy_path], source: "budget_management_endpoints.py:215", rationale: "Budget enumeration"} diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index ed5c657d23e..016f3b7be41 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -66,6 +66,38 @@ def test_gemini_passthrough_nonstreaming_logs_cost( assert tag in (row.request_tags or []), f"tags not logged: {row.request_tags}" +def test_gemini_passthrough_returns_the_same_header_contract_as_the_managed_route( + client: PassthroughClient, scoped_key: str +) -> None: + """A native passthrough call must still be costable and pace-able by the client. + + Customers front provider-native traffic through /gemini/ and read the same + operational headers they get on /chat/completions: the response cost, so the + call reconciles against spend, and the x-ratelimit-* pacing headers, so a + client knows how much budget it has left. The passthrough route currently + returns neither, which makes native traffic invisible to the same tooling. + """ + result = client.gemini_generate( + scoped_key, "gemini-2.5-flash", f"Say hello in one word. {unique_marker()}" + ) + require_successful_call(result) + + assert result.call_id, "passthrough must stamp x-litellm-call-id" + assert result.response_cost is not None, ( + "passthrough generateContent returned no x-litellm-response-cost header, so a " + "native call cannot be reconciled against spend the way /chat/completions can" + ) + assert result.response_cost > 0, ( + f"x-litellm-response-cost must be a real cost, got {result.response_cost}" + ) + + pacing = tuple(name for name in result.headers if name.startswith("x-ratelimit-")) + assert pacing, ( + "passthrough generateContent returned no x-ratelimit-* headers, so a client " + f"cannot pace itself; headers present were {sorted(result.headers)}" + ) + + def test_gemini_passthrough_streaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: diff --git a/tests/e2e/management/test_budget_customer_user_org_e2e.py b/tests/e2e/management/test_budget_customer_user_org_e2e.py index 54cc18b228b..2fb52ecbe66 100644 --- a/tests/e2e/management/test_budget_customer_user_org_e2e.py +++ b/tests/e2e/management/test_budget_customer_user_org_e2e.py @@ -22,7 +22,7 @@ import pytest from pydantic import BaseModel, RootModel from e2e_config import unique_marker -from e2e_http import NoBody, unwrap +from e2e_http import NoBody, is_ok, unwrap from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, OrgInfoParams, OrgNewBody, UserNewBody @@ -53,9 +53,15 @@ class BudgetNewResponse(BaseModel): budget_id: str +class ModelBudgetEntry(BaseModel): + budget_limit: float + time_period: str + + class BudgetUpdateBody(BaseModel): budget_id: str - max_budget: float + max_budget: float | None = None + model_max_budget: dict[str, ModelBudgetEntry] | None = None class BudgetInfoBody(BaseModel): @@ -66,6 +72,7 @@ class BudgetRow(BaseModel): budget_id: str | None = None max_budget: float | None = None soft_budget: float | None = None + model_max_budget: dict[str, ModelBudgetEntry] | None = None class BudgetInfoResponse(RootModel[list[BudgetRow]]): @@ -148,6 +155,56 @@ class TestBudgetManagement: f"/budget/list never included the created budget {budget_id}", ) + @pytest.mark.covers("mgmt.budget.update.accepts_model_max_budget") + def test_update_accepts_per_model_budgets_including_punctuated_names( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """Per-model caps must be settable on an existing budget. + + `model_max_budget` is how a customer caps spend per model on a shared + budget, and model ids routinely carry dots and hyphens (`glm-5.2`). The + route has to accept both, and the plain name is included so a failure + says whether per-model budgets are broken outright or only for + punctuated ids. + """ + for model_name in ("gpt4o", "glm-5.2"): + budget_id = _create_budget( + client, resources, BudgetNewBody(max_budget=_INITIAL_MAX_BUDGET) + ) + + result = client.proxy.transport.post( + "/budget/update", + headers=client.proxy.transport.master, + json=BudgetUpdateBody( + budget_id=budget_id, + model_max_budget={ + model_name: ModelBudgetEntry(budget_limit=5.0, time_period="1d") + }, + ), + response_type=NoBody, + ) + + assert is_ok(result), ( + f"/budget/update rejected a per-model budget for {model_name!r}: {result}; " + f"a customer cannot cap spend per model on an existing budget" + ) + + def has_model_budget() -> BudgetRow | None: + row = next( + (r for r in _budget_rows(client, budget_id) if r.budget_id == budget_id), + None, + ) + if row is None or not row.model_max_budget: + return None + return row if model_name in row.model_max_budget else None + + _ = _poll( + client, + has_model_budget, + f"/budget/info never reported a model_max_budget entry for {model_name!r} " + f"on budget {budget_id}", + ) + @pytest.mark.covers("mgmt.budget.update.persists") def test_update_max_budget_persists_to_budget_info( self, client: ManagementClient, resources: ResourceManager