mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
828fb3e909
Review follow-ups on the contract suite. The restore case only checked the email, so a regression that left <PHONE_NUMBER_2> in the answer passed. It now requires every masked value back and rejects any numbered token by shape rather than by name. Its poll returned as soon as an email placeholder appeared, which read as accepting an unrestored response. The condition is now "the model echoed the line in either form", and the docstring says why that is the right stopping point: the restore runs per request, so a token that survived it is a verdict and not something a retry fixes. The unreachable case claimed to check that the failure does not name the endpoint and never asserted it. It does now, and the unreachable base moved from a loopback address to a host in the reserved .invalid domain, because the proxy already rewrites IPs out of error messages: against 127.0.0.1 that assertion could not fail even with the sanitizer removed. Verified by putting the raw exception back into the analyzer error, which now turns the case red and previously did not. A hostname is also what a real deployment looks like. The registry entries used assertion names outside the documented guardrail grammar. Rather than rename behaviors that are genuinely distinct, the grammar now groups the names in use, including logs_masked_entities, which was already in the registry and already outside it. The exclusion pattern is back to two explicit filenames, so a future self-contained presidio suite is not silently kept off the lane.
45 lines
1.7 KiB
Python
45 lines
1.7 KiB
Python
import re
|
|
import sys
|
|
from typing import Final
|
|
|
|
SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
|
|
UNSUPPORTED: Final = re.compile(
|
|
r"^tests/e2e/(ui|claude_code|load)/"
|
|
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
|
|
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
|
|
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
|
r"|^tests/e2e/guardrails/test_presidio_contract_e2e\.py$"
|
|
)
|
|
HARNESS: Final = re.compile(
|
|
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
|
r"|^tests/e2e/gateway/"
|
|
r"|^\.github/e2e-stack/"
|
|
r"|^\.github/workflows/test-e2e-changed\.yml$"
|
|
)
|
|
UNEXPANDED: Final = re.compile(r"[*?\[]")
|
|
|
|
|
|
def is_selectable(path: str) -> bool:
|
|
return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
|
|
|
|
|
|
def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
|
|
direct: Final = frozenset(path for path in changed if is_selectable(path))
|
|
harness_changed: Final = any(HARNESS.match(path) for path in changed)
|
|
canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
|
|
return tuple(sorted(direct | canary_tests))
|
|
|
|
|
|
def main() -> int:
|
|
canary: Final = tuple(sys.argv[1:])
|
|
unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
|
|
if unexpanded:
|
|
_ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
|
|
return 1
|
|
changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
|
|
_ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|