litellm/.github/diagnostics/rc108-integration.patch

138 lines
6.8 KiB
Diff

diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh
index 1380b07de5..8c8f1b81bf 100644
--- a/.circleci/scripts/run_integration.sh
+++ b/.circleci/scripts/run_integration.sh
@@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
test -f "$LITELLM_UI_PATH/index.html"
fi
-export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
+export INTEGRATION_SEED=2659939133
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@@ -254,11 +254,17 @@ elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
fi
+diagnostic_tests=()
+if [ "$suite" = extensions ]; then
+ diagnostic_tests=(tests/integration/observability/test_logging_only_scope_chaos.py::test_K4_proxy_restart_after_fifteen_responses_records_lost_ids)
+fi
+
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_RUN_ID="$integration_identity" \
DATABASE_URL="$DATABASE_URL" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
+ PYTEST_ADDOPTS="-o junit_logging=all" \
LITELLM_MODE=PRODUCTION \
"${license_env[@]}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
@@ -270,7 +276,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
- "${shard_arguments[@]}"
+ "${shard_arguments[@]}" "${diagnostic_tests[@]}"
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
for covered_pid in "$proxy_pid" "$peer_pid"; do
diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py
index e752c835e4..3a2e31cb0a 100644
--- a/tests/integration/management/test_team_delete_large_membership.py
+++ b/tests/integration/management/test_team_delete_large_membership.py
@@ -10,6 +10,8 @@ from __future__ import annotations
import json
import os
+import time
+import traceback
import uuid
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
@@ -24,6 +26,7 @@ import yaml
from pydantic import JsonValue
from redis import Redis
+from tests.integration._support.budget import case_timeout_seconds
from tests.integration._support.client import (
Gateway,
Scenario,
@@ -732,16 +735,52 @@ def _statuses(proxy: Gateway, keys: Sequence[str]) -> dict[int, int]:
return {status: statuses.count(status) for status in frozenset(statuses)}
+@pytest.mark.timeout(case_timeout_seconds() * 2)
def test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer(
- gateway: Gateway, peer: Gateway
+ gateway: Gateway, peer: Gateway, record_property: Callable[[str, object], None]
) -> None:
with gateway.scenario() as scenario:
team: Final = _create_team(scenario)
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
keys: Final = _seeded_team_keys(team, KEYS_WELL_PAST_THE_PUBLISH_CAP)
+ warm_started: Final = time.monotonic()
assert _statuses(peer, keys) == {200: len(keys)}
-
- response: Final = _delete_teams(gateway, [team])
+ record_property("diagnostic_warm_seconds", time.monotonic() - warm_started)
+ delete_started: Final = time.monotonic()
+ try:
+ response: Final = _delete_teams(gateway, [team])
+ except httpx.TimeoutException:
+ print(traceback.format_exc(), flush=True)
+ snapshot: Final = read_rows(
+ "SELECT pid, state, wait_event_type, wait_event, "
+ "extract(epoch FROM clock_timestamp() - xact_start)::float AS xact_seconds, "
+ "pg_blocking_pids(pid) AS blockers, left(query, 3000) AS query "
+ "FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() ORDER BY pid",
+ (),
+ )
+ record_property("diagnostic_initial_delete_timeout_seconds", time.monotonic() - delete_started)
+ record_property("diagnostic_database_activity", json.dumps(snapshot))
+ print(json.dumps({"diagnostic": "initial_delete_timeout", "team": team, "activity": snapshot}), flush=True)
+ still_present: Final = eventually(
+ lambda: bool(_team_rows(team)), lambda present: not present, seconds=75, return_last_on_timeout=True
+ )
+ record_property("diagnostic_team_present_after_observation", still_present)
+ record_property("diagnostic_observation_seconds", time.monotonic() - delete_started)
+ remaining: Final = read_rows(
+ 'SELECT count(*)::int AS count FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)
+ )
+ record_property("diagnostic_remaining_keys", json.dumps(remaining))
+ if not still_present:
+ revoked: Final = _statuses(peer, keys)
+ record_property("diagnostic_peer_key_statuses", json.dumps(revoked))
+ print(
+ json.dumps(
+ {"diagnostic": "after_initial_delete", "team_present": still_present, "remaining_keys": remaining}
+ ),
+ flush=True,
+ )
+ raise
+ record_property("diagnostic_delete_seconds", time.monotonic() - delete_started)
assert response.status_code == 200, response.text
assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) == []
diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py
index be38d5a6f1..a3f9b9e57a 100644
--- a/tests/integration/observability/test_logging_only_scope_chaos.py
+++ b/tests/integration/observability/test_logging_only_scope_chaos.py
@@ -13,6 +13,7 @@ from typing import Final
import httpx
import psutil
import pytest
+from tests.integration._support.budget import case_timeout_seconds
from _logging_only_scope_support import (
JSON_OBJECT,
CallerResult,
@@ -568,7 +569,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows(
)
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(case_timeout_seconds() * 2)
def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
gateway: Gateway,
tmp_path: Path,