mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
138 lines
6.8 KiB
Diff
138 lines
6.8 KiB
Diff
diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh
|
|
index 1380b07de5..8c8f1b81bf 100644
|
|
--- a/.circleci/scripts/run_integration.sh
|
|
+++ b/.circleci/scripts/run_integration.sh
|
|
@@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
|
|
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
|
|
test -f "$LITELLM_UI_PATH/index.html"
|
|
fi
|
|
-export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
|
|
+export INTEGRATION_SEED=2659939133
|
|
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
|
|
|
|
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
|
|
@@ -254,11 +254,17 @@ elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
|
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
|
|
fi
|
|
|
|
+diagnostic_tests=()
|
|
+if [ "$suite" = extensions ]; then
|
|
+ diagnostic_tests=(tests/integration/observability/test_logging_only_scope_chaos.py::test_K4_proxy_restart_after_fifteen_responses_records_lost_ids)
|
|
+fi
|
|
+
|
|
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|
INTEGRATION_RUN_ID="$integration_identity" \
|
|
DATABASE_URL="$DATABASE_URL" \
|
|
"${shared_services[@]}" "${worker_coverage[@]}" \
|
|
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
|
+ PYTEST_ADDOPTS="-o junit_logging=all" \
|
|
LITELLM_MODE=PRODUCTION \
|
|
"${license_env[@]}" \
|
|
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
|
@@ -270,7 +276,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
|
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
|
|
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
|
|
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
|
|
- "${shard_arguments[@]}"
|
|
+ "${shard_arguments[@]}" "${diagnostic_tests[@]}"
|
|
|
|
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
|
for covered_pid in "$proxy_pid" "$peer_pid"; do
|
|
diff --git a/tests/integration/management/test_team_delete_large_membership.py b/tests/integration/management/test_team_delete_large_membership.py
|
|
index e752c835e4..3a2e31cb0a 100644
|
|
--- a/tests/integration/management/test_team_delete_large_membership.py
|
|
+++ b/tests/integration/management/test_team_delete_large_membership.py
|
|
@@ -10,6 +10,8 @@ from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
+import time
|
|
+import traceback
|
|
import uuid
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
@@ -24,6 +26,7 @@ import yaml
|
|
from pydantic import JsonValue
|
|
from redis import Redis
|
|
|
|
+from tests.integration._support.budget import case_timeout_seconds
|
|
from tests.integration._support.client import (
|
|
Gateway,
|
|
Scenario,
|
|
@@ -732,16 +735,52 @@ def _statuses(proxy: Gateway, keys: Sequence[str]) -> dict[int, int]:
|
|
return {status: statuses.count(status) for status in frozenset(statuses)}
|
|
|
|
|
|
+@pytest.mark.timeout(case_timeout_seconds() * 2)
|
|
def test_deleting_a_team_with_more_keys_than_the_publish_cap_revokes_every_key_on_a_warmed_peer(
|
|
- gateway: Gateway, peer: Gateway
|
|
+ gateway: Gateway, peer: Gateway, record_property: Callable[[str, object], None]
|
|
) -> None:
|
|
with gateway.scenario() as scenario:
|
|
team: Final = _create_team(scenario)
|
|
scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
|
|
keys: Final = _seeded_team_keys(team, KEYS_WELL_PAST_THE_PUBLISH_CAP)
|
|
+ warm_started: Final = time.monotonic()
|
|
assert _statuses(peer, keys) == {200: len(keys)}
|
|
-
|
|
- response: Final = _delete_teams(gateway, [team])
|
|
+ record_property("diagnostic_warm_seconds", time.monotonic() - warm_started)
|
|
+ delete_started: Final = time.monotonic()
|
|
+ try:
|
|
+ response: Final = _delete_teams(gateway, [team])
|
|
+ except httpx.TimeoutException:
|
|
+ print(traceback.format_exc(), flush=True)
|
|
+ snapshot: Final = read_rows(
|
|
+ "SELECT pid, state, wait_event_type, wait_event, "
|
|
+ "extract(epoch FROM clock_timestamp() - xact_start)::float AS xact_seconds, "
|
|
+ "pg_blocking_pids(pid) AS blockers, left(query, 3000) AS query "
|
|
+ "FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() ORDER BY pid",
|
|
+ (),
|
|
+ )
|
|
+ record_property("diagnostic_initial_delete_timeout_seconds", time.monotonic() - delete_started)
|
|
+ record_property("diagnostic_database_activity", json.dumps(snapshot))
|
|
+ print(json.dumps({"diagnostic": "initial_delete_timeout", "team": team, "activity": snapshot}), flush=True)
|
|
+ still_present: Final = eventually(
|
|
+ lambda: bool(_team_rows(team)), lambda present: not present, seconds=75, return_last_on_timeout=True
|
|
+ )
|
|
+ record_property("diagnostic_team_present_after_observation", still_present)
|
|
+ record_property("diagnostic_observation_seconds", time.monotonic() - delete_started)
|
|
+ remaining: Final = read_rows(
|
|
+ 'SELECT count(*)::int AS count FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)
|
|
+ )
|
|
+ record_property("diagnostic_remaining_keys", json.dumps(remaining))
|
|
+ if not still_present:
|
|
+ revoked: Final = _statuses(peer, keys)
|
|
+ record_property("diagnostic_peer_key_statuses", json.dumps(revoked))
|
|
+ print(
|
|
+ json.dumps(
|
|
+ {"diagnostic": "after_initial_delete", "team_present": still_present, "remaining_keys": remaining}
|
|
+ ),
|
|
+ flush=True,
|
|
+ )
|
|
+ raise
|
|
+ record_property("diagnostic_delete_seconds", time.monotonic() - delete_started)
|
|
assert response.status_code == 200, response.text
|
|
assert read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) == []
|
|
|
|
diff --git a/tests/integration/observability/test_logging_only_scope_chaos.py b/tests/integration/observability/test_logging_only_scope_chaos.py
|
|
index be38d5a6f1..a3f9b9e57a 100644
|
|
--- a/tests/integration/observability/test_logging_only_scope_chaos.py
|
|
+++ b/tests/integration/observability/test_logging_only_scope_chaos.py
|
|
@@ -13,6 +13,7 @@ from typing import Final
|
|
import httpx
|
|
import psutil
|
|
import pytest
|
|
+from tests.integration._support.budget import case_timeout_seconds
|
|
from _logging_only_scope_support import (
|
|
JSON_OBJECT,
|
|
CallerResult,
|
|
@@ -568,7 +569,7 @@ def test_K3_two_worker_sigkill_checks_post_kill_spend_rows(
|
|
)
|
|
|
|
|
|
-@pytest.mark.timeout(180)
|
|
+@pytest.mark.timeout(case_timeout_seconds() * 2)
|
|
def test_K4_proxy_restart_after_fifteen_responses_records_lost_ids(
|
|
gateway: Gateway,
|
|
tmp_path: Path,
|