test(e2e): strengthen shallow MODERATE assertions to catch regressions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/e2e/batches/test_batches_e2e.py

@@ -777,9 +777,10 @@ class TestGeminiFiles:
         resources.defer(lambda: client.delete_model(model_id))
         key = resources.key()
 
+        payload = render_jsonl(GEMINI_FILES_RAW_MODEL)
         file = unwrap(
             client.upload_file(
-                content=render_jsonl(GEMINI_FILES_RAW_MODEL),
+                content=payload,
                 form=FileUploadForm(purpose="batch", target_model_names=model_name),
                 key=key,
             )
@@ -787,6 +788,9 @@ class TestGeminiFiles:
         resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
         assert_file_object(file, provider="gemini")
         assert file.id, "gemini file upload returned no id"
+        assert file.bytes == len(payload), (
+            f"gemini stored {file.bytes} bytes for a {len(payload)}-byte upload; the file was truncated or misrecorded"
+        )
 
 
 def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMParamsBody:

tests/e2e/llm_translation/test_chat_completions_regression_e2e.py

@@ -111,6 +111,25 @@ def _assert_describes_cat(response: ChatResponse) -> None:
     )
 
 
+def _assert_says_pong(response: ChatResponse, provider: str) -> None:
+    assert response.model, f"{provider}: response carried no model name: {response}"
+    assert response.choices, f"{provider}: chat returned no choices: {response}"
+    message = response.choices[0].message
+    content = (message.content if message else None) or ""
+    assert "pong" in content.lower(), (
+        f"{provider}: expected the model to reply 'pong', got {content[:200]!r}"
+    )
+
+
+def _assert_costed(client: PassthroughClient, key: str, provider: str) -> None:
+    rows = client.proxy.poll_logs_for_key(
+        key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs)
+    )
+    priced = [r for r in rows if (r.spend or 0) > 0]
+    assert priced, f"{provider} chat was not costed on key ...{key[-6:]}: {rows}"
+    assert priced[0].status == "success", f"{provider} chat spend status={priced[0].status!r}"
+
+
 def _streamed_text(events: list[str]) -> str:
     """Concatenate the delta content across streamed chunks. Parsing every event as
     JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image
@@ -274,9 +293,8 @@ class TestCohereChat:
                 ),
             )
         )
-        assert response.choices, f"cohere chat returned no choices: {response}"
-        content = response.choices[0].message.content if response.choices[0].message else None
-        assert content and content.strip(), f"cohere empty content: {response}"
+        _assert_says_pong(response, "cohere")
+        _assert_costed(client, key, "cohere")
 
 
 class TestGeminiChatCompletions:
@@ -375,9 +393,7 @@ class TestHostedVllmChat:
                 ),
             )
         )
-        assert response.choices, f"hosted_vllm chat returned no choices: {response}"
-        content = response.choices[0].message.content if response.choices[0].message else None
-        assert content and content.strip(), f"hosted_vllm empty content: {response}"
+        _assert_says_pong(response, "hosted_vllm")
 
 
 class TestOpenAIChatCompletions:
@@ -671,9 +687,8 @@ class TestBedrockConverseChatCompletions:
                 ),
             )
         )
-        assert response.choices, f"bedrock converse chat returned no choices: {response}"
-        content = response.choices[0].message.content if response.choices[0].message else None
-        assert content and content.strip(), f"bedrock converse returned empty content: {response}"
+        _assert_says_pong(response, "bedrock converse")
+        _assert_costed(client, key, "bedrock converse")
 
     @pytest.mark.covers(
         "llm.chat_completions.bedrock_converse.basic.stream.works",

tests/e2e/llm_translation/test_responses_e2e.py

@@ -261,10 +261,19 @@ class TestResponses:
         resources.defer(lambda: endpoints_client.delete_model(model_id))
         key = resources.key()
 
-        result = endpoints_client.responses(key, model, "reply with one word")
+        result = endpoints_client.responses(key, model, f"reply with one word {unique_marker()}")
         require_successful_call(result)
         parsed = ResponsesResult.model_validate_json(result.body)
         assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}"
+        assert parsed.id and parsed.model, f"/responses over bedrock missing identifiers: {result.body[:300]}"
+
+        rows = endpoints_client.proxy.poll_logs_for_request_id(
+            parsed.id,
+            predicate=lambda logged_rows: any((row.spend or 0) > 0 for row in logged_rows),
+        )
+        row = next((logged_row for logged_row in rows if (logged_row.spend or 0) > 0), None)
+        assert row is not None, f"no costed spend row for bedrock response id {parsed.id}"
+        assert "claude" in (row.model or "").lower(), f"unexpected bedrock spend row model: {row.model}"
 
     @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
     def test_responses_bedrock_returns_function_call(

tests/e2e/management/management_client.py

@@ -274,12 +274,14 @@ class ManagementClient:
             )
         ).user_id
 
-    def create_customer(self, user_id: str) -> str:
+    def create_customer(
+        self, user_id: str, *, alias: str | None = None, max_budget: float | None = None
+    ) -> str:
         _ = unwrap(
             self.proxy.transport.post(
                 "/customer/new",
                 headers=self.proxy.transport.master,
-                json=CustomerNewBody(user_id=user_id),
+                json=CustomerNewBody(user_id=user_id, alias=alias, max_budget=max_budget),
                 response_type=CustomerResponse,
             )
         )

tests/e2e/management/test_management_e2e.py

@@ -618,10 +618,18 @@ class TestCustomer:
         self, client: ManagementClient, resources: ResourceManager
     ) -> None:
         customer = f"e2e-customer-{unique_marker()}"
-        client.create_customer(customer)
+        alias = f"alias-{unique_marker()}"
+        client.create_customer(customer, alias=alias, max_budget=12.5)
         resources.defer(lambda: client.delete_customer(customer))
 
         info = client.customer_info(customer)
         assert info.user_id == customer, (
             f"/customer/info did not report the created end-user; got {info.user_id!r}"
         )
+        assert info.alias == alias, (
+            f"/customer/info dropped the alias set on create; got {info.alias!r}"
+        )
+        assert info.litellm_budget_table is not None and info.litellm_budget_table.max_budget == 12.5, (
+            f"/customer/info did not persist the customer's max_budget; got {info.litellm_budget_table!r}"
+        )
+        assert (info.spend or 0) == 0, f"a freshly created customer must start at zero spend; got {info.spend!r}"

tests/e2e/models.py

@@ -117,10 +117,19 @@ class KeyInfoResponse(BaseModel):
 
 class CustomerNewBody(BaseModel):
     user_id: str
+    alias: str | None = None
+    max_budget: float | None = None
+
+
+class CustomerBudget(BaseModel):
+    max_budget: float | None = None
 
 
 class CustomerResponse(BaseModel):
     user_id: str | None = None
+    alias: str | None = None
+    spend: float | None = None
+    litellm_budget_table: CustomerBudget | None = None
 
 
 class CustomerInfoParams(BaseModel):