scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

test(e2e): repair fallback management test for config-owned fallbacks (#47023)

Browse source 
* test(e2e): repair fallback round trip for config-owned fallbacks

The e2e stack config sets router_settings.fallbacks, and since #45860 a
/fallback write to a key the config file owns is refused with a 400. The
round trip now runs on context_window fallbacks, which the stack config
leaves to the database, and also deletes the rule and checks it is gone.

A second test locks the new contract: a /fallback write to the
config-owned fallbacks key returns 400 naming router_settings.fallbacks,
and the config rule keeps being served unchanged.

* test(e2e): assert the refused fallback rule is never served
This commit is contained in:
yuneng-jiang 2026-10-16 18:27:39 -07:00 • committed by GitHub
parent 856a3f62f3
commit 59fa785de0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 133 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
tests/e2e/coverage_registry/mgmt.yaml
View file

@ -78,6 +78,7 @@
- {id: mgmt.compliance.gdpr.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "compliance_endpoints.py", rationale: "GDPR ops (smoke)"}
- {id: mgmt.tool_management.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "tool_management_endpoints.py", rationale: "Tool inventory (smoke)"}
- {id: mgmt.fallback_management.update.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "fallback_management_endpoints.py", rationale: "Fallback config (smoke)"}
- {id: mgmt.fallback_management.update.config_owned_refused, module: mgmt, tier: P1, surface: api, assertions: [config_owned_refused], source: "fallback_management_endpoints.py", rationale: "A /fallback write to a fallbacks key the config file sets is refused and the config rule keeps routing"}
- {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
- {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
- {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}

155
tests/e2e/management/test_config_misc_endpoints_e2e.py
View file

@ -31,7 +31,7 @@ import pytest
from pydantic import BaseModel
from e2e_config import unique_marker
from e2e_http import NoBody, Success, unwrap, unwrap_status
from e2e_http import NoBody, Result, Success, UnknownApiError, unwrap, unwrap_status
from e2e_metadata import Domain, Route, Subject, meta
from lifecycle import ResourceManager
from management_client import ManagementClient
@ -141,12 +141,15 @@ class ComplianceResponse(BaseModel):
# ---- fallback management ---------------------------------------------------
class FallbackShape(BaseModel):
class FallbackTarget(BaseModel):
model: str
fallback_models: list[str]
fallback_type: str
class FallbackShape(FallbackTarget):
fallback_models: list[str]
class FallbackCreateBody(FallbackShape):
pass
@ -163,6 +166,28 @@ class FallbackGetResponse(FallbackShape):
pass
class FallbackDeleteResponse(FallbackTarget):
message: str
class RouterFallbackValues(BaseModel):
fallbacks: list[dict[str, list[str]]] | None = None
class RouterFallbackOwnership(BaseModel):
current_values: RouterFallbackValues
source: dict[str, str]
class ConfigOwnedDetail(BaseModel):
keys: list[str]
section: str
class ConfigOwnedRejection(BaseModel):
detail: ConfigOwnedDetail
# ---- jwt key mapping -------------------------------------------------------
@ -365,20 +390,19 @@ class TestFallbackManagement:
@pytest.mark.covers("mgmt.fallback_management.update.happy_path")
@meta(Subject(domain=Domain.ROUTING))
def test_create_persists_and_is_read_back(self, client: ManagementClient, resources: ResourceManager) -> None:
primary = f"e2e-fallback-primary-{unique_marker()}"
secondary = f"e2e-fallback-secondary-{unique_marker()}"
params = LiteLLMParamsBody(model="openai/gpt-5.5", api_key="e2e-dummy-key")
primary_id = client.proxy.create_model(primary, params)
resources.defer(lambda: client.proxy.delete_model(primary_id))
secondary_id = client.proxy.create_model(secondary, params)
resources.defer(lambda: client.proxy.delete_model(secondary_id))
resources.defer(lambda: self._delete_fallback(client, primary))
ownership = self._router_fallback_ownership(client)
assert ownership.source.get("context_window_fallbacks") != "config", (
"the proxy config sets router_settings.context_window_fallbacks, so /fallback cannot write it and this "
"round trip needs a fallback type the database owns"
)
primary, secondary = self._register_fallback_pair(client, resources)
resources.defer(lambda: self._delete_fallback(client, primary, "context_window"))
created = unwrap(
client.proxy.transport.post(
"/fallback",
headers=client.proxy.transport.master,
json=FallbackCreateBody(model=primary, fallback_models=[secondary], fallback_type="general"),
json=FallbackCreateBody(model=primary, fallback_models=[secondary], fallback_type="context_window"),
response_type=FallbackResponse,
)
)
@ -388,13 +412,7 @@ class TestFallbackManagement:
)
def read_back() -> FallbackGetResponse | None:
result = client.proxy.transport.get(
f"/fallback/{primary}",
headers=client.proxy.transport.master,
params=FallbackGetParams(fallback_type="general"),
response_type=FallbackGetResponse,
)
match result:
match self._get_fallback(client, primary, "context_window"):
case Success(data=data) if secondary in data.fallback_models:
return data
case _:
@ -405,14 +423,105 @@ class TestFallbackManagement:
f"GET /fallback/{primary} reports fallbacks {got.fallback_models}, configured [{secondary!r}]"
)
deleted = unwrap(self._delete_fallback(client, primary, "context_window"))
assert deleted.model == primary and deleted.fallback_type == "context_window", (
f"DELETE /fallback/{primary} answered for model={deleted.model!r} type={deleted.fallback_type!r}"
)
def gone() -> bool | None:
match self._get_fallback(client, primary, "context_window"):
case UnknownApiError(status_code=404):
return True
case _:
return None
_ = _poll(client, gone, f"GET /fallback/{primary} still reports a fallback after DELETE /fallback/{primary}")
@pytest.mark.covers("mgmt.fallback_management.update.config_owned_refused")
@meta(Subject(domain=Domain.ROUTING))
def test_write_to_config_owned_fallbacks_is_refused(
self, client: ManagementClient, resources: ResourceManager
) -> None:
before = self._router_fallback_ownership(client)
config_rules = before.current_values.fallbacks
assert before.source.get("fallbacks") == "config" and config_rules, (
"this test needs router_settings.fallbacks set in the proxy config, but /router/settings reports "
f"source={before.source.get('fallbacks')!r} value={config_rules}"
)
primary, secondary = self._register_fallback_pair(client, resources)
resources.defer(lambda: self._delete_fallback(client, primary, "general"))
result = client.proxy.transport.post(
"/fallback",
headers=client.proxy.transport.master,
json=FallbackCreateBody(model=primary, fallback_models=[secondary], fallback_type="general"),
response_type=FallbackResponse,
)
match result:
case UnknownApiError(status_code=400, body=body):
rejection = ConfigOwnedRejection.model_validate_json(body).detail
case _:
pytest.fail(f"/fallback accepted a write to the config-owned router_settings.fallbacks: {result}")
assert rejection.keys == ["fallbacks"] and rejection.section == "router_settings", (
f"/fallback refused the write naming keys={rejection.keys} section={rejection.section!r}, "
"expected router_settings.fallbacks"
)
after = self._router_fallback_ownership(client).current_values.fallbacks
assert after == config_rules, (
f"the refused write changed router_settings.fallbacks from {config_rules} to {after}"
)
match self._get_fallback(client, primary, "general"):
case UnknownApiError(status_code=404):
pass
case refused_rule:
pytest.fail(f"the refused rule for {primary} is served anyway: {refused_rule}")
config_model, config_targets = next(iter(config_rules[0].items()))
served = unwrap(self._get_fallback(client, config_model, "general"))
assert served.fallback_models == config_targets, (
f"GET /fallback/{config_model} reports {served.fallback_models}, "
f"the config file routes it to {config_targets}"
)
@staticmethod
def _delete_fallback(client: ManagementClient, model: str) -> None:
_ = client.proxy.transport.delete(
def _register_fallback_pair(client: ManagementClient, resources: ResourceManager) -> tuple[str, str]:
primary = f"e2e-fallback-primary-{unique_marker()}"
secondary = f"e2e-fallback-secondary-{unique_marker()}"
params = LiteLLMParamsBody(model="openai/gpt-5.5", api_key="e2e-dummy-key")
primary_id = client.proxy.create_model(primary, params)
resources.defer(lambda: client.proxy.delete_model(primary_id))
secondary_id = client.proxy.create_model(secondary, params)
resources.defer(lambda: client.proxy.delete_model(secondary_id))
return primary, secondary
@staticmethod
def _router_fallback_ownership(client: ManagementClient) -> RouterFallbackOwnership:
return unwrap(
client.proxy.transport.get(
"/router/settings",
headers=client.proxy.transport.master,
params=NoBody(),
response_type=RouterFallbackOwnership,
)
)
@staticmethod
def _get_fallback(client: ManagementClient, model: str, fallback_type: str) -> Result[FallbackGetResponse]:
return client.proxy.transport.get(
f"/fallback/{model}",
headers=client.proxy.transport.master,
params=FallbackGetParams(fallback_type=fallback_type),
response_type=FallbackGetResponse,
)
@staticmethod
def _delete_fallback(client: ManagementClient, model: str, fallback_type: str) -> Result[FallbackDeleteResponse]:
return client.proxy.transport.delete(
f"/fallback/{model}",
headers=client.proxy.transport.master,
json=NoBody(),
params=FallbackGetParams(fallback_type="general"),
response_type=NoBody,
params=FallbackGetParams(fallback_type=fallback_type),
response_type=FallbackDeleteResponse,
)