mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(integration): diagnose rc qualification failures
This commit is contained in:
parent
f91774e2f9
commit
6551f2c659
2 changed files with 147 additions and 1 deletions
134
.github/diagnostics/rc108-integration.patch
vendored
Normal file
134
.github/diagnostics/rc108-integration.patch
vendored
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh
|
||||
index 1380b07de5..64b492015e 100644
|
||||
--- a/.circleci/scripts/run_integration.sh
|
||||
+++ b/.circleci/scripts/run_integration.sh
|
||||
@@ -254,6 +254,13 @@ elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
||||
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
|
||||
fi
|
||||
|
||||
+diagnostic_tests=()
|
||||
+if [ "$suite" = management ]; then
|
||||
+ diagnostic_tests=(tests/integration/management/test_team_delete_large_membership.py::test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer tests/integration/management/test_user_rate_limit_updates.py::test_user_rate_limit_explicit_null_clears_and_omitted_limit_is_untouched 'tests/integration/management/test_team_update_multi_instance.py::test_team_update_adds_model_on_peer_for_inference_routes[responses]')
|
||||
+elif [ "$suite" = extensions ]; then
|
||||
+ diagnostic_tests=(tests/integration/observability/test_logging_only_scope_chaos.py::test_K4_proxy_restart_after_fifteen_responses_records_lost_ids)
|
||||
+fi
|
||||
+
|
||||
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
||||
INTEGRATION_RUN_ID="$integration_identity" \
|
||||
DATABASE_URL="$DATABASE_URL" \
|
||||
@@ -270,7 +277,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
||||
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
|
||||
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
|
||||
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
|
||||
- "${shard_arguments[@]}"
|
||||
+ "${shard_arguments[@]}" "${diagnostic_tests[@]}"
|
||||
|
||||
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
||||
for covered_pid in "$proxy_pid" "$peer_pid"; do
|
||||
diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py
|
||||
index e752c835e4..c6de799d0f 100644
|
||||
--- a/tests/integration/management/test_team_delete_large_membership.py
|
||||
+++ b/tests/integration/management/test_team_delete_large_membership.py
|
||||
@@ -10,6 +10,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
+import time
|
||||
+import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
@@ -732,16 +734,69 @@ def _statuses(proxy: Gateway, keys: Sequence[str]) -> dict[int, int]:
|
||||
return {status: statuses.count(status) for status in frozenset(statuses)}
|
||||
|
||||
|
||||
+@pytest.mark.parametrize("diagnostic_repeat", range(20))
|
||||
def test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer(
|
||||
- gateway: Gateway, peer: Gateway
|
||||
+ gateway: Gateway, peer: Gateway, diagnostic_repeat: int
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
team: Final = _create_team(scenario)
|
||||
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
|
||||
keys: Final = _seeded_team_keys(team, KEYS_WELL_PAST_THE_PUBLISH_CAP)
|
||||
+ warm_started: Final = time.monotonic()
|
||||
assert _statuses(peer, keys) == {200: len(keys)}
|
||||
-
|
||||
- response: Final = _delete_teams(gateway, [team])
|
||||
+ print(
|
||||
+ json.dumps(
|
||||
+ {"diagnostic": "warm_complete", "repeat": diagnostic_repeat, "seconds": time.monotonic() - warm_started}
|
||||
+ ),
|
||||
+ flush=True,
|
||||
+ )
|
||||
+ delete_started: Final = time.monotonic()
|
||||
+ try:
|
||||
+ response: Final = _delete_teams(gateway, [team])
|
||||
+ except httpx.TimeoutException:
|
||||
+ print(traceback.format_exc(), flush=True)
|
||||
+ snapshot: Final = read_rows(
|
||||
+ "SELECT pid, state, wait_event_type, wait_event, "
|
||||
+ "extract(epoch FROM clock_timestamp() - xact_start)::float AS xact_seconds, "
|
||||
+ "pg_blocking_pids(pid) AS blockers, left(query, 3000) AS query "
|
||||
+ "FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() ORDER BY pid",
|
||||
+ (),
|
||||
+ )
|
||||
+ print(
|
||||
+ json.dumps(
|
||||
+ {
|
||||
+ "diagnostic": "initial_delete_timeout",
|
||||
+ "team": team,
|
||||
+ "seconds": time.monotonic() - delete_started,
|
||||
+ "activity": snapshot,
|
||||
+ }
|
||||
+ ),
|
||||
+ flush=True,
|
||||
+ )
|
||||
+ still_present: Final = eventually(
|
||||
+ lambda: bool(_team_rows(team)), lambda present: not present, seconds=75, return_last_on_timeout=True
|
||||
+ )
|
||||
+ print(
|
||||
+ json.dumps(
|
||||
+ {
|
||||
+ "diagnostic": "after_initial_delete",
|
||||
+ "team_present": still_present,
|
||||
+ "seconds": time.monotonic() - delete_started,
|
||||
+ }
|
||||
+ ),
|
||||
+ flush=True,
|
||||
+ )
|
||||
+ raise
|
||||
+ print(
|
||||
+ json.dumps(
|
||||
+ {
|
||||
+ "diagnostic": "delete_complete",
|
||||
+ "repeat": diagnostic_repeat,
|
||||
+ "seconds": time.monotonic() - delete_started,
|
||||
+ }
|
||||
+ ),
|
||||
+ flush=True,
|
||||
+ )
|
||||
assert response.status_code == 200, response.text
|
||||
assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) == []
|
||||
|
||||
diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py
|
||||
index be38d5a6f1..a3f9b9e57a 100644
|
||||
--- a/tests/integration/observability/test_logging_only_scope_chaos.py
|
||||
+++ b/tests/integration/observability/test_logging_only_scope_chaos.py
|
||||
@@ -13,6 +13,7 @@ from typing import Final
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
+from tests.integration._support.budget import case_timeout_seconds
|
||||
from _logging_only_scope_support import (
|
||||
JSON_OBJECT,
|
||||
CallerResult,
|
||||
@@ -568,7 +569,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows(
|
||||
)
|
||||
|
||||
|
||||
-@pytest.mark.timeout(180)
|
||||
+@pytest.mark.timeout(case_timeout_seconds() * 2)
|
||||
def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
|
||||
gateway: Gateway,
|
||||
tmp_path: Path,
|
||||
14
.github/workflows/test-integration.yml
vendored
14
.github/workflows/test-integration.yml
vendored
|
|
@ -48,7 +48,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["management","extensions"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
|
|
@ -81,6 +81,18 @@ jobs:
|
|||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: 61fe5be5b89df971a97be580f6f2d6d18615810d
|
||||
|
||||
- name: Apply RC integration diagnostics
|
||||
env:
|
||||
DIAGNOSTIC_SOURCE_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
[[ "$DIAGNOSTIC_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
git fetch --depth 1 origin "$DIAGNOSTIC_SOURCE_SHA"
|
||||
git show "$DIAGNOSTIC_SOURCE_SHA:.github/diagnostics/rc108-integration.patch" > "$RUNNER_TEMP/rc108-integration.patch"
|
||||
git apply --check "$RUNNER_TEMP/rc108-integration.patch"
|
||||
git apply "$RUNNER_TEMP/rc108-integration.patch"
|
||||
git diff --exit-code HEAD -- litellm enterprise litellm-proxy-extras
|
||||
|
||||
- name: Set up Python
|
||||
timeout-minutes: 3
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue