From 5c6854510400ca53a8718401adf0ca911f65e62a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 21 Sep 2026 15:00:54 -0700 Subject: [PATCH] test: fix stale budget-status and bad-database-url assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI checks were asserting behaviour the proxy no longer has. Neither was catching anything; both now fail for the right reason. budget_exceeded (tests/otel_tests/test_e2e_budgeting.py) bf804f5188 made 422 the default for budget refusals and added budget_exceeded_status_code to restore 429 for callers that need it. The e2e budget tests still asserted 429, so all six have been failing on a status change that was deliberate. Assert 422, the documented default, rather than reading litellm.budget_exceeded_status_code back — a test that asks the code what it does would have passed straight through this change and through the next one. The helpers also caught bare Exception, so a connection error reached `e.body` and surfaced as an AttributeError instead of a failed assertion. Narrow both to openai.APIStatusError, which is what a refusal actually raises (UnprocessableEntityError for 422, RateLimitError for 429), and let anything else propagate as itself. test_bad_database_url (.circleci/config.yml) The check required "Database setup failed after multiple retries", which only the v1 resolver emits, OR uvicorn's "Application startup failed. Exiting.". With v2 the default, the first branch is dead and the whole assertion rests on an incidental uvicorn line that HEAD's run did not emit at all. Assert the behaviour instead of the wording: the container exits non-zero, the log names the unreachable server (P1001), it never reaches "Application startup complete", and it is not left running. The exit code was previously discarded by `|| true`, so the one thing the job most needed to check was never checked. Verified by running the bad-DATABASE_URL container: exit 3, P1001 present, no startup-complete line, container stopped — the new check passes and the old one passed only by the uvicorn line's accident. --- .circleci/config.yml | 23 ++++++++++++----------- tests/otel_tests/test_e2e_budgeting.py | 13 +++++++------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cc9aa7fe1c4..c69ed6a3510 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3033,28 +3033,29 @@ jobs: - run: name: Run Docker container with bad DATABASE_URL command: | + set +e docker run --name my-app \ -p 4000:4000 \ -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ - --port 4000 > docker_output.log 2>&1 || true + --port 4000 > docker_output.log 2>&1 + echo "$?" > docker_exit_code + set -e - run: name: Display Docker logs command: cat docker_output.log - run: - name: Check for expected error + name: Proxy must refuse to serve on an unreachable database command: | - if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ - (grep -q "Database setup failed after multiple retries" docker_output.log || \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then - echo "Expected error found. Test passed." - else - echo "Expected error not found. Test failed." - cat docker_output.log - exit 1 - fi + fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } + exit_code="$(cat docker_exit_code)" + [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" + grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" + ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" + ! docker exec my-app true 2>/dev/null || fail "container is still running" + echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." provider_replay_harness: docker: diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index ca5058818e4..ae8f0ddc3ec 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -5,6 +5,7 @@ import uuid from typing import Any, Optional import aiohttp +import openai import pytest from httpx import AsyncClient @@ -23,7 +24,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k call_count += 1 await asyncio.sleep(0.1) # allow spend tracking to catch up pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls") - except Exception as e: + except openai.APIStatusError as e: print("vars: ", vars(e)) print("e.body: ", e.body) @@ -32,8 +33,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "429" - ), f"Expected error code 429, got: {error_dict['code']}" + error_dict["code"] == "422" + ), f"Expected error code 422, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" @@ -506,9 +507,9 @@ async def make_calls_until_team_budget_exceeded_cli_sso( call_count += 1 await asyncio.sleep(0.1) pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls") - except Exception as e: + except openai.APIStatusError as e: error_dict = e.body - assert error_dict["code"] == "429" + assert error_dict["code"] == "422" assert error_dict["type"] == "budget_exceeded" message = error_dict["message"] assert "Budget has been exceeded!" in message @@ -556,7 +557,7 @@ async def test_team_budget_enforcement_cli_sso_token(): 1. Create team with a tiny max_budget and a user on that team 2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint) 3. Make chat completion calls until the team budget is exceeded - 4. Verify HTTP 429 budget_exceeded names the team + 4. Verify HTTP 422 budget_exceeded names the team """ user_id = f"cli-budget-user-{uuid.uuid4().hex[:8]}" user_email = f"{user_id}@example.com"