test(e2e): mark batches suite with coverage registry cells

Wire @pytest.mark.covers so non_core_llms counts the existing batches
lifecycle, key-access-denied, and file upload/delete tests against the
registry cells in llm_nonconversational.yaml
This commit is contained in:
mubashir1osmani 2026-07-16 16:48:46 -07:00
parent c18816cd0d
commit ad57c5d1c2
2 changed files with 57 additions and 1 deletions

View file

@ -180,3 +180,43 @@ def matches_id_shape(shape: IdShape, id_str: str) -> bool:
if shape == "model_encoded":
return is_model_encoded_id(id_str)
return not is_managed_id(id_str) and not is_model_encoded_id(id_str)
def coverage_cells_for_lifecycle(cap: Capability) -> tuple[str, ...]:
"""Registry cell ids that the parametrized lifecycle test covers for one capability.
OpenAI has per-scenario cells plus granular create/retrieve/cancel/list/file
cells. Other providers have one basic cell each. File-upload cells for the
batch-backing path are included when the lifecycle uploads for that provider.
"""
match cap.provider:
case "openai":
cells = (
f"llm.batches.openai_{cap.scenario}.basic.nonstream.works",
"llm.batches.openai.create.nonstream.works",
"llm.batches.openai.retrieve.nonstream.works",
"llm.batches.openai.file_lifecycle.nonstream.works",
"llm.files.openai.upload.nonstream.works",
)
if cap.can_cancel:
cells = (*cells, "llm.batches.openai.cancel.nonstream.works")
if cap.can_list:
cells = (*cells, "llm.batches.openai.list.nonstream.works")
return cells
case "azure":
return (
"llm.batches.azure_openai.basic.nonstream.works",
"llm.files.azure_openai.upload.nonstream.works",
)
case "vertex_ai":
return (
"llm.batches.vertex.basic.nonstream.works",
"llm.files.vertex.upload.nonstream.works",
)
case "bedrock":
return (
"llm.batches.bedrock.basic.nonstream.works",
"llm.files.bedrock.upload.nonstream.works",
)
case _:
return ()

View file

@ -37,6 +37,7 @@ from capabilities import (
CAPABILITIES,
FILE_ID_SHAPE,
Capability,
coverage_cells_for_lifecycle,
matches_id_shape,
raw_id_matches_provider,
)
@ -168,7 +169,17 @@ def assert_batch_object(batch: BatchObject) -> None:
), "batch.created_at missing"
@pytest.mark.parametrize("cap", CAPABILITIES, ids=[c.id for c in CAPABILITIES])
@pytest.mark.parametrize(
"cap",
[
pytest.param(
cap,
id=cap.id,
marks=pytest.mark.covers(*coverage_cells_for_lifecycle(cap)),
)
for cap in CAPABILITIES
],
)
def test_batch_lifecycle(
cap: Capability,
client: BatchClient,
@ -266,6 +277,7 @@ def test_batch_lifecycle(
assert match.object == "batch"
@pytest.mark.covers("llm.batches.openai.key_model_access_denied.nonstream.works")
def test_batch_key_model_access_denied(
client: BatchClient, resources: ResourceManager, batch_deployments: None
) -> None:
@ -301,6 +313,10 @@ def test_batch_key_model_access_denied(
), f"restricted key created a batch for a disallowed model (status {denied_create.status_code})"
@pytest.mark.covers(
"llm.files.openai.upload.nonstream.works",
"llm.files.openai.delete.nonstream.works",
)
def test_file_upload_and_delete_outputs(
client: BatchClient, resources: ResourceManager, batch_deployments: None
) -> None: