diff --git a/tests/llm_translation/reasoning_effort_grid/grid_spec.py b/tests/llm_translation/reasoning_effort_grid/grid_spec.py index 3eaa9bc2950..51a027ec101 100644 --- a/tests/llm_translation/reasoning_effort_grid/grid_spec.py +++ b/tests/llm_translation/reasoning_effort_grid/grid_spec.py @@ -22,6 +22,7 @@ class ModelEntry: required_env: FrozenSet[str] = field(default_factory=frozenset) caps: FrozenSet[str] = field(default_factory=frozenset) unavailable_error: Optional[str] = None + fail_reason: Optional[str] = None bedrock_effort_ceiling: Optional[str] = None def params(self) -> Dict[str, str]: @@ -126,7 +127,7 @@ _VERTEX_REQ = frozenset({"VERTEX_PROJECT"}) _BEDROCK_REQ = frozenset({"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"}) -_CAPS_OPUS_4_7: FrozenSet[str] = frozenset( +_CAPS_XHIGH_MAX: FrozenSet[str] = frozenset( {"supports_xhigh_reasoning_effort", "supports_max_reasoning_effort"} ) _CAPS_4_6: FrozenSet[str] = frozenset({"supports_max_reasoning_effort"}) @@ -134,12 +135,19 @@ _CAPS_NONE: FrozenSet[str] = frozenset() ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="claude-opus-4-8", + model="anthropic/claude-opus-4-8", + mode="adaptive", + required_env=_ANTHROPIC_REQ, + caps=_CAPS_XHIGH_MAX, + ), ModelEntry( alias="claude-opus-4-7", model="anthropic/claude-opus-4-7", mode="adaptive", required_env=_ANTHROPIC_REQ, - caps=_CAPS_OPUS_4_7, + caps=_CAPS_XHIGH_MAX, ), ModelEntry( alias="claude-sonnet-4-6", @@ -159,12 +167,25 @@ ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = ( AZURE_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="azure-claude-opus-4-8", + model="azure_ai/claude-opus-4-8", + mode="adaptive", + required_env=_AZURE_FOUNDRY_REQ, + caps=_CAPS_XHIGH_MAX, + fail_reason=( + "claude-opus-4-8 has no deployment on the CI Microsoft Foundry " + "resource yet; Foundry returns DeploymentNotFound until someone " + "creates the opus-4-8 deployment, so this cell stays loud in CI. " + "Remove this fail_reason once the deployment exists." + ), + ), ModelEntry( alias="azure-claude-opus-4-7", model="azure_ai/claude-opus-4-7", mode="adaptive", required_env=_AZURE_FOUNDRY_REQ, - caps=_CAPS_OPUS_4_7, + caps=_CAPS_XHIGH_MAX, ), ModelEntry( alias="azure-claude-opus-4-6", @@ -191,13 +212,27 @@ AZURE_AI_MODELS: Tuple[ModelEntry, ...] = ( VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="vertex-claude-opus-4-8", + model="vertex_ai/claude-opus-4-8", + mode="adaptive", + extra_params=(("vertex_location", "global"),), + required_env=_VERTEX_REQ, + caps=_CAPS_XHIGH_MAX, + fail_reason=( + "claude-opus-4-8 availability on the CI Vertex project is not yet " + "confirmed for this brand-new release, so this cell stays loud in " + "CI until verified. Remove this fail_reason once the model is " + "confirmed available on the global Vertex endpoint." + ), + ), ModelEntry( alias="vertex-claude-opus-4-7", model="vertex_ai/claude-opus-4-7", mode="adaptive", extra_params=(("vertex_location", "global"),), required_env=_VERTEX_REQ, - caps=_CAPS_OPUS_4_7, + caps=_CAPS_XHIGH_MAX, ), ModelEntry( alias="vertex-claude-opus-4-6", @@ -227,13 +262,24 @@ VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = ( BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = ( + ModelEntry( + alias="bedrock-claude-opus-4-8", + model="bedrock/converse/us.anthropic.claude-opus-4-8", + mode="adaptive", + extra_params=(("aws_region_name", "us-east-1"),), + required_env=_BEDROCK_REQ, + caps=_CAPS_XHIGH_MAX, + bedrock_effort_ceiling="xhigh", + unavailable_error="is not available for this account", + ), ModelEntry( alias="bedrock-claude-opus-4-7", model="bedrock/converse/us.anthropic.claude-opus-4-7", mode="adaptive", extra_params=(("aws_region_name", "us-east-1"),), required_env=_BEDROCK_REQ, - caps=_CAPS_OPUS_4_7, + caps=_CAPS_XHIGH_MAX, + bedrock_effort_ceiling="xhigh", unavailable_error="is not available for this account", ), ModelEntry( diff --git a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py index b4382e16fed..a5ef1b7ea3c 100644 --- a/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py +++ b/tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py @@ -173,6 +173,9 @@ async def test_reasoning_effort_grid( if skip_reason: pytest.skip(skip_reason) + if model.fail_reason: + pytest.xfail(model.fail_reason) + if route_name == "bedrock_invoke_messages": status, exc = await _call_messages(model, effort) else: @@ -197,8 +200,8 @@ async def test_reasoning_effort_grid( def test_grid_cell_count() -> None: - assert len(_PARAMS) == 21 * 11, ( - f"expected 231 cells (21 provider x model combos x 11 efforts), " + assert len(_PARAMS) == 25 * 11, ( + f"expected 275 cells (25 provider x model combos x 11 efforts), " f"got {len(_PARAMS)}" )