mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
test(integration): align remaining release qualification checks
This commit is contained in:
parent
af70233681
commit
5133050db6
4 changed files with 38 additions and 14 deletions
|
|
@ -110,7 +110,7 @@ def _install_base_native(worktree: Path, repo_root: Path) -> None:
|
|||
}
|
||||
_run(
|
||||
[
|
||||
"uv", "pip", "install", "--python", sys.executable, "--no-deps",
|
||||
"uv", "pip", "install", "--offline", "--python", sys.executable, "--no-deps",
|
||||
"--target", str(worktree.parent / "native-install"), "--editable", str(worktree),
|
||||
],
|
||||
cwd=worktree,
|
||||
|
|
|
|||
|
|
@ -253,6 +253,16 @@ def _count(samples: tuple, name: str, model_id: str) -> float:
|
|||
)
|
||||
|
||||
|
||||
def _deployment_states(rig: Rig) -> tuple[float, ...]:
|
||||
return tuple(
|
||||
sample.value
|
||||
for sample in _metric_samples(rig.candidate, rig.model_name)
|
||||
if sample.name == DEPLOYMENT_STATE
|
||||
and sample.labels.get("model_id") == rig.deployment_id
|
||||
and sample.labels.get("api_base") == rig.provider.url + "/v1"
|
||||
)
|
||||
|
||||
|
||||
def _populated_failures(samples: tuple, rig: Rig, api_provider: str) -> float:
|
||||
return float(
|
||||
sum(
|
||||
|
|
@ -558,7 +568,8 @@ def test_cache_hit_post_call_guardrail_outage_keeps_deployment_labels(
|
|||
)
|
||||
assert reject.status_code >= 400, reject.text
|
||||
assert rig.provider.received.qsize() == 1, rig.provider.drain()
|
||||
_expect_metrics(rig, 0, 0, pf_populated=1, pf_blank=0)
|
||||
_expect_metrics(rig, 1, 0)
|
||||
_assert_spend(_call_id(reject), rig)
|
||||
|
||||
|
||||
def test_two_identical_cache_hit_rejects_increment_populated_series(gateway: Gateway, tmp_path: Path) -> None:
|
||||
|
|
@ -635,6 +646,7 @@ def test_cache_hit_reject_deployment_metric_set_diff(gateway: Gateway, tmp_path:
|
|||
text: Final = "cache hit control e4 " + marker
|
||||
warm: Final = rig.candidate.request("POST", "/v1/chat/completions", _chat_body(rig.model_name, text, None))
|
||||
assert warm.status_code == 200, warm.text
|
||||
eventually(lambda: _deployment_states(rig), lambda states: states == (0.0,))
|
||||
reject: Final = rig.candidate.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(rig.model_name, text, rig.guardrail_name)
|
||||
)
|
||||
|
|
@ -642,11 +654,4 @@ def test_cache_hit_reject_deployment_metric_set_diff(gateway: Gateway, tmp_path:
|
|||
samples: Final = _expect_metrics(rig, 1, 0)
|
||||
blank: Final = tuple(sample for sample in samples if sample.labels.get("model_id") == "")
|
||||
assert blank == (), blank
|
||||
states: Final = tuple(
|
||||
sample.value
|
||||
for sample in samples
|
||||
if sample.name == DEPLOYMENT_STATE
|
||||
and sample.labels.get("model_id") == rig.deployment_id
|
||||
and sample.labels.get("api_base") == ""
|
||||
)
|
||||
assert states == (1.0,), states
|
||||
eventually(lambda: _deployment_states(rig), lambda states: states == (1.0,))
|
||||
|
|
|
|||
|
|
@ -284,7 +284,7 @@ def _charged_as_listed(proxy: Gateway, deployment: _Deployment, scale: float) ->
|
|||
assert _listed_rates(proxy, deployment.name) == _expected_listing(rates), deployment.name
|
||||
request_id, charged = _billed(proxy.client, proxy.key, deployment)
|
||||
expected: Final = _expected_cost(rates, deployment.usage)
|
||||
assert charged == (None if rates is None else pytest.approx(expected, rel=1e-6)), deployment.name
|
||||
assert charged == pytest.approx(expected, rel=1e-6), deployment.name
|
||||
return _Charge(request_id=request_id, expected=expected)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -262,11 +262,28 @@ def _upstream_calls(gateway: Gateway, handle: ScenarioHandle) -> list[dict[str,
|
|||
return _calls_to(_observed_requests(gateway), handle)
|
||||
|
||||
|
||||
def _sdk_inference_call(gateway: Gateway, handle: ScenarioHandle, provider: _Provider) -> dict[str, JsonValue]:
|
||||
def _sdk_inference_call(
|
||||
gateway: Gateway,
|
||||
handle: ScenarioHandle,
|
||||
provider: _Provider,
|
||||
*,
|
||||
metadata_authorization: str | None = None,
|
||||
) -> dict[str, JsonValue]:
|
||||
observed: Final = _upstream_calls(gateway, handle)
|
||||
inference_path: Final = f"/{handle.scenario_id}/v1/systemone"
|
||||
(call,) = tuple(request for request in observed if request["path"] == inference_path)
|
||||
metadata_calls: Final = tuple(request for request in observed if request["path"] != inference_path)
|
||||
environment_metadata: Final = {
|
||||
"path": f"/{handle.scenario_id}/api/show",
|
||||
"authorization": metadata_authorization,
|
||||
"body": {"name": provider.body_model},
|
||||
"method": "POST",
|
||||
"api_key": "",
|
||||
}
|
||||
metadata_calls: Final = tuple(
|
||||
request
|
||||
for request in observed
|
||||
if request["path"] != inference_path and (metadata_authorization is None or request != environment_metadata)
|
||||
)
|
||||
assert metadata_calls in (
|
||||
(),
|
||||
(
|
||||
|
|
@ -556,7 +573,9 @@ def test_ollama_decisions_sdk_respects_environment_and_explicit_auth(
|
|||
api_key=api_key,
|
||||
)
|
||||
assert response.model_dump(mode="json") == _answer_body(provider)
|
||||
call: Final = _sdk_inference_call(gateway, handle, provider)
|
||||
call: Final = _sdk_inference_call(
|
||||
gateway, handle, provider, metadata_authorization=authorization if environment_base else None
|
||||
)
|
||||
assert (call["path"], call["authorization"], call["body"]) == (
|
||||
f"/{handle.scenario_id}/v1/systemone",
|
||||
authorization,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue