From 7dc7cde5a64f8f53da182431b705b5a7da560d2a Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Wed, 22 Jul 2026 05:57:37 +0000 Subject: [PATCH] test(e2e): strengthen shallow MODERATE assertions to catch regressions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/batches/test_batches_e2e.py | 6 +++- .../test_chat_completions_regression_e2e.py | 33 ++++++++++++++----- .../e2e/llm_translation/test_responses_e2e.py | 11 ++++++- tests/e2e/management/management_client.py | 6 ++-- tests/e2e/management/test_management_e2e.py | 10 +++++- tests/e2e/models.py | 9 +++++ 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index b0c53becb6b..c6e8a846746 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -777,9 +777,10 @@ class TestGeminiFiles: resources.defer(lambda: client.delete_model(model_id)) key = resources.key() + payload = render_jsonl(GEMINI_FILES_RAW_MODEL) file = unwrap( client.upload_file( - content=render_jsonl(GEMINI_FILES_RAW_MODEL), + content=payload, form=FileUploadForm(purpose="batch", target_model_names=model_name), key=key, ) @@ -787,6 +788,9 @@ class TestGeminiFiles: resources.defer(quietly(lambda: client.delete_file(file.id, key=key))) assert_file_object(file, provider="gemini") assert file.id, "gemini file upload returned no id" + assert file.bytes == len(payload), ( + f"gemini stored {file.bytes} bytes for a {len(payload)}-byte upload; the file was truncated or misrecorded" + ) def _vllm_params(api_base: str, api_key: str | None, model_id: str) -> LiteLLMParamsBody: diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 8d3622e441a..add93bd35b9 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -111,6 +111,25 @@ def _assert_describes_cat(response: ChatResponse) -> None: ) +def _assert_says_pong(response: ChatResponse, provider: str) -> None: + assert response.model, f"{provider}: response carried no model name: {response}" + assert response.choices, f"{provider}: chat returned no choices: {response}" + message = response.choices[0].message + content = (message.content if message else None) or "" + assert "pong" in content.lower(), ( + f"{provider}: expected the model to reply 'pong', got {content[:200]!r}" + ) + + +def _assert_costed(client: PassthroughClient, key: str, provider: str) -> None: + rows = client.proxy.poll_logs_for_key( + key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs) + ) + priced = [r for r in rows if (r.spend or 0) > 0] + assert priced, f"{provider} chat was not costed on key ...{key[-6:]}: {rows}" + assert priced[0].status == "success", f"{provider} chat spend status={priced[0].status!r}" + + def _streamed_text(events: list[str]) -> str: """Concatenate the delta content across streamed chunks. Parsing every event as JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image @@ -274,9 +293,8 @@ class TestCohereChat: ), ) ) - assert response.choices, f"cohere chat returned no choices: {response}" - content = response.choices[0].message.content if response.choices[0].message else None - assert content and content.strip(), f"cohere empty content: {response}" + _assert_says_pong(response, "cohere") + _assert_costed(client, key, "cohere") class TestGeminiChatCompletions: @@ -375,9 +393,7 @@ class TestHostedVllmChat: ), ) ) - assert response.choices, f"hosted_vllm chat returned no choices: {response}" - content = response.choices[0].message.content if response.choices[0].message else None - assert content and content.strip(), f"hosted_vllm empty content: {response}" + _assert_says_pong(response, "hosted_vllm") class TestOpenAIChatCompletions: @@ -671,9 +687,8 @@ class TestBedrockConverseChatCompletions: ), ) ) - assert response.choices, f"bedrock converse chat returned no choices: {response}" - content = response.choices[0].message.content if response.choices[0].message else None - assert content and content.strip(), f"bedrock converse returned empty content: {response}" + _assert_says_pong(response, "bedrock converse") + _assert_costed(client, key, "bedrock converse") @pytest.mark.covers( "llm.chat_completions.bedrock_converse.basic.stream.works", diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index d24d2b53b71..c4e1c4c78a8 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -261,10 +261,19 @@ class TestResponses: resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() - result = endpoints_client.responses(key, model, "reply with one word") + result = endpoints_client.responses(key, model, f"reply with one word {unique_marker()}") require_successful_call(result) parsed = ResponsesResult.model_validate_json(result.body) assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}" + assert parsed.id and parsed.model, f"/responses over bedrock missing identifiers: {result.body[:300]}" + + rows = endpoints_client.proxy.poll_logs_for_request_id( + parsed.id, + predicate=lambda logged_rows: any((row.spend or 0) > 0 for row in logged_rows), + ) + row = next((logged_row for logged_row in rows if (logged_row.spend or 0) > 0), None) + assert row is not None, f"no costed spend row for bedrock response id {parsed.id}" + assert "claude" in (row.model or "").lower(), f"unexpected bedrock spend row model: {row.model}" @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") def test_responses_bedrock_returns_function_call( diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index cdc31aeea79..b863fa3a55b 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -274,12 +274,14 @@ class ManagementClient: ) ).user_id - def create_customer(self, user_id: str) -> str: + def create_customer( + self, user_id: str, *, alias: str | None = None, max_budget: float | None = None + ) -> str: _ = unwrap( self.proxy.transport.post( "/customer/new", headers=self.proxy.transport.master, - json=CustomerNewBody(user_id=user_id), + json=CustomerNewBody(user_id=user_id, alias=alias, max_budget=max_budget), response_type=CustomerResponse, ) ) diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 9b398963ac9..090b177cfda 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -618,10 +618,18 @@ class TestCustomer: self, client: ManagementClient, resources: ResourceManager ) -> None: customer = f"e2e-customer-{unique_marker()}" - client.create_customer(customer) + alias = f"alias-{unique_marker()}" + client.create_customer(customer, alias=alias, max_budget=12.5) resources.defer(lambda: client.delete_customer(customer)) info = client.customer_info(customer) assert info.user_id == customer, ( f"/customer/info did not report the created end-user; got {info.user_id!r}" ) + assert info.alias == alias, ( + f"/customer/info dropped the alias set on create; got {info.alias!r}" + ) + assert info.litellm_budget_table is not None and info.litellm_budget_table.max_budget == 12.5, ( + f"/customer/info did not persist the customer's max_budget; got {info.litellm_budget_table!r}" + ) + assert (info.spend or 0) == 0, f"a freshly created customer must start at zero spend; got {info.spend!r}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index b3ea9346180..147e11639a9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -117,10 +117,19 @@ class KeyInfoResponse(BaseModel): class CustomerNewBody(BaseModel): user_id: str + alias: str | None = None + max_budget: float | None = None + + +class CustomerBudget(BaseModel): + max_budget: float | None = None class CustomerResponse(BaseModel): user_id: str | None = None + alias: str | None = None + spend: float | None = None + litellm_budget_table: CustomerBudget | None = None class CustomerInfoParams(BaseModel):