mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-04 02:31:27 +00:00
fix(e2e): keep the CLI determinism test out of the in-cluster suite
It drives the real CLI for several seconds. The edge stamps every upstream call with PYTEST_CURRENT_TEST, a process-global that names whichever test the worker is in when the call arrives rather than the one that made it, so a test that holds a worker that long collects other tests' in-flight calls. Build 234's key report credits this test with 20 Bedrock and 7 Anthropic misses, and it makes no provider call at all. Those misattributed calls take the wrong test id into the cache key and write recordings under it, so the test was polluting the shared corpus it exists to protect. Deselected unless E2E_CLI_DETERMINISM is set, the same opt-in shape the managed-files, prompt-caching and redis-chaos markers already use. The attribution bug itself is older than this branch and is reported, not fixed here.
This commit is contained in:
parent
b9751a38ab
commit
afb28540bb
4 changed files with 11 additions and 0 deletions
|
|
@ -36,6 +36,8 @@ import pytest
|
|||
from claude_code.cli_driver import _FIXED_CLI_USER_ID, _seed_cli_identity, _stable_cli_state, run_claude
|
||||
from claude_code.rate_limiter import RateLimiter
|
||||
|
||||
pytestmark = pytest.mark.cli_determinism
|
||||
|
||||
_STUB_REPLY = {
|
||||
"id": "msg_stub",
|
||||
"type": "message",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from typing import Final
|
|||
import pytest
|
||||
import requests
|
||||
from e2e_config import (
|
||||
CLI_DETERMINISM_OPT_IN_ENV,
|
||||
CONTROL_PLANE_BASE_URL,
|
||||
FIXTURE_DIR,
|
||||
FIXTURE_MODE_RAW,
|
||||
|
|
@ -53,6 +54,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
|
|||
"managed_files": MANAGED_FILES_OPT_IN_ENV,
|
||||
"prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV,
|
||||
"redis_chaos": REDIS_CHAOS_OPT_IN_ENV,
|
||||
"cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -120,6 +122,11 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including "
|
||||
"prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"cli_determinism: drives the real claude CLI for several seconds, which widens the window in which "
|
||||
"another test's in-flight upstream call is attributed to it; deselected unless E2E_CLI_DETERMINISM is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from "
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
|
|||
MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
|
||||
PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK"
|
||||
REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS"
|
||||
CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
|
||||
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
|
||||
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
|
||||
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
|
||||
|
|
|
|||
|
|
@ -10,4 +10,5 @@ markers =
|
|||
weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set
|
||||
managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set
|
||||
prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set
|
||||
cli_determinism: drives the real claude CLI for several seconds, which widens the window in which another test's in-flight upstream call is attributed to it; deselected unless E2E_CLI_DETERMINISM is set
|
||||
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue