test: repair four chronically failing CI tests

test_no_linear_scans_in_router: #39468 added config_deployments() and
heuristic_v2_router_limit_violation(), which both scan the whole model_list
from admin-only paths (model add/upsert), so add them to the allowlist. The
allowlist becomes a mapping so each exemption carries its reason as data.

test_missing_model_parameter_curl: a request with no model is rejected by the
proxy when nothing can serve it and by the router when a wildcard or default
deployment exists, and by the upstream provider when a wildcard forwards it,
so the message text is not a stable contract. Assert the contract that holds
in every case: HTTP 400 with a non-empty error message.

test_model_group_info_e2e: /model_group/info resolves wildcards, so it can
never return "anthropic/*" verbatim. cc3f9cd65b rewrote the assertion to
expect the raw pattern after claude-3-5-haiku-20241022 left the price map,
which made it unsatisfiable. Assert the expansion instead.

test_should_derive_ocr_mapping_status_from_live_tests: the audit needs a
native bridge built with the trace-parity feature, which CI never builds, so
skip with the harness's own diagnostic instead of erroring. Extract that
check out of ensure_trace_bridge as trace_bridge_error so a pytest run
reports the state without kicking off a maturin rebuild.
This commit is contained in:
Yuneng Jiang 2026-09-04 10:10:47 -07:00
parent 2e734004f9
commit dbf8fe0f4e
No known key found for this signature in database
5 changed files with 82 additions and 22 deletions

View file

@ -237,10 +237,12 @@ class TestRouterIndexManagement:
- model_name_to_deployment_indices for O(1) + O(k) model_name lookups
"""
# Methods that are allowed to iterate through self.model_list
ALLOWED_METHODS = [
"_get_deployment_by_litellm_model", # Edge case: lookup by litellm_params.model (not indexed)
"_finalize_adaptive_router_if_configured", # Init-time prefix scan for "auto_router/adaptive_router" (no index for prefix match)
]
ALLOWED_METHODS = {
"_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed",
"_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match',
"config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)",
"heuristic_v2_router_limit_violation": "counts heuristic_v2 routers across the whole list; admin path only (auto-router init/upsert)",
}
# Get path to router.py
router_file = os.path.join(

View file

@ -73,6 +73,16 @@ def _rebuild(repo_root: Path) -> tuple[bool, str]:
return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:])
def trace_bridge_error() -> str | None:
"""Why the installed bridge cannot serve trace parity, or None when it can. Never rebuilds."""
bridge: Final = get_native_bridge()
if bridge is None:
return "native Rust bridge is not importable"
if getattr(bridge, "_trace", None) is None:
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
return None
def ensure_trace_bridge(repo_root: Path) -> str | None:
native_path: Final = _native_module_path()
native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None
@ -84,9 +94,4 @@ def ensure_trace_bridge(repo_root: Path) -> str | None:
if not succeeded:
return f"native Rust bridge rebuild failed:\n{output}"
_drop_imported_bridge()
bridge: Final = get_native_bridge()
if bridge is None:
return "native Rust bridge is not importable"
if getattr(bridge, "_trace", None) is None:
return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature"
return None
return trace_bridge_error()

View file

@ -106,15 +106,22 @@ def test_missing_model_parameter_curl(curl_command):
# Run the curl command and capture the output
key = generate_key_sync()
curl_command = curl_command.replace("sk-1234", key)
result = subprocess.run(curl_command, shell=True, capture_output=True, text=True)
result = subprocess.run(
f'{curl_command} -s -w "\\n%{{http_code}}"',
shell=True,
capture_output=True,
text=True,
)
body, _, status_code = result.stdout.rpartition("\n")
# Parse the JSON response
response = json.loads(result.stdout)
response = json.loads(body)
# Check that we got an error response
assert "error" in response
print("error in response", json.dumps(response, indent=4))
assert "litellm.BadRequestError" in response["error"]["message"]
assert status_code == "400", f"expected HTTP 400, got {status_code}: {response}"
assert isinstance(response["error"]["message"], str) and response["error"]["message"]
@pytest.mark.asyncio

View file

@ -487,6 +487,9 @@ async def test_get_personal_models_for_user():
async def test_model_group_info_e2e():
"""
Test /model/group/info endpoint
The proxy config declares a wildcard "anthropic/*" deployment, and the endpoint resolves
wildcards into the concrete models they cover, so the raw pattern is never returned.
"""
async with aiohttp.ClientSession() as session:
models = await get_models(session=session, key="sk-1234")
@ -495,16 +498,13 @@ async def test_model_group_info_e2e():
model_group_info = await get_model_group_info(session=session, key="sk-1234")
print(model_group_info)
# Check that the endpoint returns data and contains the wildcard
# anthropic model group from the proxy config
has_anthropic_wildcard = False
for model in model_group_info["data"]:
if model["model_group"] == "anthropic/*":
has_anthropic_wildcard = True
model_groups = [m["model_group"] for m in model_group_info["data"]]
assert has_anthropic_wildcard, (
f"Expected 'anthropic/*' in model groups, got: "
f"{[m['model_group'] for m in model_group_info['data']]}"
assert "anthropic/*" not in model_groups, (
f"Expected 'anthropic/*' to be expanded, but it was returned verbatim: {model_groups}"
)
assert any(m.startswith("anthropic/") for m in model_groups), (
f"Expected concrete anthropic models from the 'anthropic/*' config entry, got: {model_groups}"
)

View file

@ -2,6 +2,7 @@ from __future__ import annotations
import importlib
from pathlib import Path
from types import SimpleNamespace
from typing import Final
import pytest
@ -13,6 +14,7 @@ mapping_validator = importlib.import_module("tests.rust-python-harness.strategie
mappings = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mappings")
ocr_mapping = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.cases.ocr")
cli = importlib.import_module("tests.rust-python-harness.cli")
native_build = importlib.import_module("tests.rust-python-harness.shared.native_build")
audit_mapping = mapping_validator.audit_mapping
UNIT_TEST_CONTRACTS = mappings.UNIT_TEST_CONTRACTS
@ -119,7 +121,51 @@ def test_should_leave_functions_without_mapping_contracts_unimplemented() -> Non
assert "messages" not in UNIT_TEST_CONTRACTS
def test_should_report_a_bridge_that_cannot_be_imported() -> None:
with pytest.MonkeyPatch.context() as patch:
patch.setattr(native_build, "get_native_bridge", lambda: None)
message = native_build.trace_bridge_error()
assert message is not None
assert "not importable" in message
def test_should_report_a_bridge_built_without_the_trace_feature() -> None:
with pytest.MonkeyPatch.context() as patch:
patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=None))
message = native_build.trace_bridge_error()
assert message is not None
assert native_build.BRIDGE_FEATURE in message
def test_should_accept_a_bridge_built_with_the_trace_feature() -> None:
with pytest.MonkeyPatch.context() as patch:
patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object()))
assert native_build.trace_bridge_error() is None
def test_should_not_rebuild_the_bridge_while_reporting_its_state() -> None:
rebuilds: list[object] = []
def fake_rebuild(repo_root: object) -> tuple[bool, str]:
rebuilds.append(repo_root)
return True, ""
with pytest.MonkeyPatch.context() as patch:
patch.setattr(native_build, "_rebuild", fake_rebuild)
patch.setattr(native_build, "get_native_bridge", lambda: None)
native_build.trace_bridge_error()
assert rebuilds == []
def test_should_derive_ocr_mapping_status_from_live_tests() -> None:
bridge_error: Final = native_build.trace_bridge_error()
if bridge_error is not None:
pytest.skip(bridge_error)
report = audit_mapping(OCR_CONTRACT, repo_root=REPO_ROOT)
assert report.is_valid, (