test(integration): diagnose rc qualification failures

This commit is contained in:
Yuneng Jiang 2026-11-02 10:57:59 -08:00
parent f91774e2f9
commit 6551f2c659
2 changed files with 147 additions and 1 deletions

View file

@ -0,0 +1,134 @@
diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh
index 1380b07de5..64b492015e 100644
--- a/.circleci/scripts/run_integration.sh
+++ b/.circleci/scripts/run_integration.sh
@@ -254,6 +254,13 @@ elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
fi
+diagnostic_tests=()
+if [ "$suite" = management ]; then
+ diagnostic_tests=(tests/integration/management/test_team_delete_large_membership.py::test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer tests/integration/management/test_user_rate_limit_updates.py::test_user_rate_limit_explicit_null_clears_and_omitted_limit_is_untouched 'tests/integration/management/test_team_update_multi_instance.py::test_team_update_adds_model_on_peer_for_inference_routes[responses]')
+elif [ "$suite" = extensions ]; then
+ diagnostic_tests=(tests/integration/observability/test_logging_only_scope_chaos.py::test_K4_proxy_restart_after_fifteen_responses_records_lost_ids)
+fi
+
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_RUN_ID="$integration_identity" \
DATABASE_URL="$DATABASE_URL" \
@@ -270,7 +277,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
- "${shard_arguments[@]}"
+ "${shard_arguments[@]}" "${diagnostic_tests[@]}"
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
for covered_pid in "$proxy_pid" "$peer_pid"; do
diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py
index e752c835e4..c6de799d0f 100644
--- a/tests/integration/management/test_team_delete_large_membership.py
+++ b/tests/integration/management/test_team_delete_large_membership.py
@@ -10,6 +10,8 @@ from __future__ import annotations
import json
import os
+import time
+import traceback
import uuid
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
@@ -732,16 +734,69 @@ def _statuses(proxy: Gateway, keys: Sequence[str]) -> dict[int, int]:
return {status: statuses.count(status) for status in frozenset(statuses)}
+@pytest.mark.parametrize("diagnostic_repeat", range(20))
def test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer(
- gateway: Gateway, peer: Gateway
+ gateway: Gateway, peer: Gateway, diagnostic_repeat: int
) -> None:
with gateway.scenario() as scenario:
team: Final = _create_team(scenario)
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
keys: Final = _seeded_team_keys(team, KEYS_WELL_PAST_THE_PUBLISH_CAP)
+ warm_started: Final = time.monotonic()
assert _statuses(peer, keys) == {200: len(keys)}
-
- response: Final = _delete_teams(gateway, [team])
+ print(
+ json.dumps(
+ {"diagnostic": "warm_complete", "repeat": diagnostic_repeat, "seconds": time.monotonic() - warm_started}
+ ),
+ flush=True,
+ )
+ delete_started: Final = time.monotonic()
+ try:
+ response: Final = _delete_teams(gateway, [team])
+ except httpx.TimeoutException:
+ print(traceback.format_exc(), flush=True)
+ snapshot: Final = read_rows(
+ "SELECT pid, state, wait_event_type, wait_event, "
+ "extract(epoch FROM clock_timestamp() - xact_start)::float AS xact_seconds, "
+ "pg_blocking_pids(pid) AS blockers, left(query, 3000) AS query "
+ "FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() ORDER BY pid",
+ (),
+ )
+ print(
+ json.dumps(
+ {
+ "diagnostic": "initial_delete_timeout",
+ "team": team,
+ "seconds": time.monotonic() - delete_started,
+ "activity": snapshot,
+ }
+ ),
+ flush=True,
+ )
+ still_present: Final = eventually(
+ lambda: bool(_team_rows(team)), lambda present: not present, seconds=75, return_last_on_timeout=True
+ )
+ print(
+ json.dumps(
+ {
+ "diagnostic": "after_initial_delete",
+ "team_present": still_present,
+ "seconds": time.monotonic() - delete_started,
+ }
+ ),
+ flush=True,
+ )
+ raise
+ print(
+ json.dumps(
+ {
+ "diagnostic": "delete_complete",
+ "repeat": diagnostic_repeat,
+ "seconds": time.monotonic() - delete_started,
+ }
+ ),
+ flush=True,
+ )
assert response.status_code == 200, response.text
assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) == []
diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py
index be38d5a6f1..a3f9b9e57a 100644
--- a/tests/integration/observability/test_logging_only_scope_chaos.py
+++ b/tests/integration/observability/test_logging_only_scope_chaos.py
@@ -13,6 +13,7 @@ from typing import Final
import httpx
import psutil
import pytest
+from tests.integration._support.budget import case_timeout_seconds
from _logging_only_scope_support import (
JSON_OBJECT,
CallerResult,
@@ -568,7 +569,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows(
)
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(case_timeout_seconds() * 2)
def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
gateway: Gateway,
tmp_path: Path,

View file

@ -48,7 +48,7 @@ jobs:
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["management","extensions"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
@ -81,6 +81,18 @@ jobs:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: 61fe5be5b89df971a97be580f6f2d6d18615810d
- name: Apply RC integration diagnostics
env:
DIAGNOSTIC_SOURCE_SHA: ${{ github.sha }}
run: |
[[ "$DIAGNOSTIC_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
git fetch --depth 1 origin "$DIAGNOSTIC_SOURCE_SHA"
git show "$DIAGNOSTIC_SOURCE_SHA:.github/diagnostics/rc108-integration.patch" > "$RUNNER_TEMP/rc108-integration.patch"
git apply --check "$RUNNER_TEMP/rc108-integration.patch"
git apply "$RUNNER_TEMP/rc108-integration.patch"
git diff --exit-code HEAD -- litellm enterprise litellm-proxy-extras
- name: Set up Python
timeout-minutes: 3