"""Owned rig and assertions for Bedrock wildcard model discovery through a scripted AWS control plane.""" from __future__ import annotations import os import re import uuid from collections import Counter from collections.abc import Iterator, Mapping from contextlib import contextmanager from pathlib import Path from typing import Final from urllib.parse import urlsplit import psutil import yaml from integration._support.aws_control_plane import ( FOUNDATION_MODELS, INFERENCE_PROFILES, Catalog, ControlPlane, ControlPlaneRequest, verified_scope, ) from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, held, object_value, string_value from integration._support.process import OwnedProxy, owned_proxy_process from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from pydantic import JsonValue HOSTED_REGIONS: Final = ("us-east-1", "us-west-2", "eu-west-1", "us-gov-west-1", "cn-north-1") UNHOSTED_REGION: Final = "eu-central-1" CONTROL_MODEL: Final = "integration-discovery-control" CONVERSE_MODEL: Final = "us.openai.gpt-5.6-sol" WORKERS: Final = 2 RELOAD_SECONDS: Final = 5 LISTING_TIMEOUT_SECONDS: Final = 10.0 PROFILE_PAGE_CAP: Final = 20 LISTING_PATHS: Final = ("/v1/models", "/models") INFO_PATHS: Final = ("/model/info", "/v1/model/info") PROFILES_QUERY: Final[Mapping[str, str]] = {"maxResults": "1000", "typeEquals": "SYSTEM_DEFINED"} FOUNDATION_QUERY: Final[Mapping[str, str]] = {"byInferenceType": "ON_DEMAND"} CLOSE: Final[Mapping[str, str]] = {"connection": "close"} LISTING_FAILURE_LOG: Final = "Error getting valid models" STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") STARTUP_COMPLETE: Final = "Application startup complete." def control_plane_host(region: str) -> str: return f"bedrock.{region}.{get_aws_dns_suffix(region)}" def credential() -> tuple[str, str]: return f"AKIA{uuid.uuid4().hex[:16].upper()}", f"secret-{uuid.uuid4().hex}" def stem() -> str: return f"disc{uuid.uuid4().hex[:10]}" def catalog_for(marker: str, *, page_size: int | None = None) -> Catalog: return Catalog( active_profiles=(f"us.{marker}.sonnet-v1:0", f"eu.{marker}.haiku-v1:0"), inactive_profiles=(f"us.{marker}.retired-v1:0",), on_demand_models=(f"{marker}.nova-micro-v1:0",), page_size=page_size, ) def discovery_config(parent: Gateway, directory: Path, *, check_provider_endpoint: bool) -> Path: configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) configuration["model_list"] = [ { "model_name": CONTROL_MODEL, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", "api_base": f"{parent.upstream_url}/v1", }, } ] configuration["litellm_settings"] = { **configuration.get("litellm_settings", {}), "check_provider_endpoint": check_provider_endpoint, } path: Final = directory / f"discovery-{'on' if check_provider_endpoint else 'off'}.yaml" path.write_text(yaml.safe_dump(configuration)) return path def discovery_environment(plane: ControlPlane, directory: Path) -> Mapping[str, str]: empty: Final = directory / "empty-aws-config" empty.write_text("") return { **plane.environment(), "AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_LOCAL_MODEL_COST_MAP": "True", "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": str(RELOAD_SECONDS), } def aws_environment_names() -> tuple[str, ...]: return tuple(name for name in os.environ if name.startswith("AWS_")) @contextmanager def discovery_proxy( parent: Gateway, directory: Path, plane: ControlPlane, *, check_provider_endpoint: bool, workers: int ) -> Iterator[OwnedProxy]: with owned_proxy_process( parent, directory, discovery_environment(plane, directory), config=discovery_config(parent, directory, check_provider_endpoint=check_provider_endpoint), remove_environment=aws_environment_names(), workers=workers, ) as owned: wait_for_every_worker(owned.log, workers) yield owned def deployment_ids(gateway: Gateway) -> frozenset[str]: entries: Final = gateway.get("/model/info")["data"] assert isinstance(entries, list) return frozenset(string_value(object_value(object_value(entry)["model_info"])["id"]) for entry in entries) def remove_deployment(gateway: Gateway, identity: str) -> None: gateway.post("/model/delete", {"id": identity}) eventually(lambda: deployment_ids(gateway), lambda ids: identity not in ids, seconds=RELOAD_SECONDS * 4) def deployment( scenario: Scenario, *, model_name: str = "bedrock/*", model: str = "bedrock/*", model_info: Mapping[str, JsonValue] | None = None, litellm_params: Mapping[str, JsonValue] | None = None, ) -> str: """A wildcard deployment posted straight to /model/new, since Scenario.model fixes a non-wildcard name.""" created: Final = scenario.gateway.post( "/model/new", { "model_name": model_name, "litellm_params": {"model": model, **(litellm_params or {})}, "model_info": dict(model_info) if model_info is not None else {}, }, ) identity: Final = string_value(object_value(created["model_info"])["id"]) scenario.cleanups.callback(remove_deployment, scenario.gateway, identity) settled_on_every_worker(scenario.gateway, identity) return identity def settled_on_every_worker(gateway: Gateway, identity: str) -> None: """Each worker picks a new deployment up on its own reload tick, so a listing straight after /model/new can land on a worker without it; the by-id lookup never lists, so holding on it leaves every worker's discovery cache cold.""" def present() -> bool: lookup: Final = gateway.request("GET", "/model/info", params={"litellm_model_id": identity}, headers=CLOSE) return lookup.status_code == 200 held(present, lambda found: found, holding=RELOAD_SECONDS * 2, seconds=RELOAD_SECONDS * 6) def sigv4_deployment( scenario: Scenario, key: str, secret: str, region: str, *, model_name: str = "bedrock/*", model: str = "bedrock/*", model_info: Mapping[str, JsonValue] | None = None, litellm_params: Mapping[str, JsonValue] | None = None, ) -> str: return deployment( scenario, model_name=model_name, model=model, model_info=model_info, litellm_params={ "aws_access_key_id": key, "aws_secret_access_key": secret, "aws_region_name": region, **(litellm_params or {}), }, ) def listed_ids( gateway: Gateway, path: str = "/v1/models", *, key: str | None = None, params: Mapping[str, str] | None = None, ) -> frozenset[str]: response: Final = gateway.request("GET", path, key=key, params=params, headers=CLOSE) assert response.status_code == 200, f"GET {path}: {response.status_code} {response.text}" data: Final = JSON_OBJECT.validate_json(response.content)["data"] assert isinstance(data, list), response.text return frozenset(string_value(object_value(entry)["id"]) for entry in data) def from_stem(marker: str, ids: frozenset[str]) -> frozenset[str]: return frozenset(name for name in ids if marker in name) def discovered( gateway: Gateway, catalog: Catalog, marker: str, path: str = "/v1/models", *, key: str | None = None, params: Mapping[str, str] | None = None, ) -> frozenset[str]: """Poll until the catalog's invocable ids are listed; returns every listed id carrying the stem.""" listed: Final = eventually( lambda: listed_ids(gateway, path, key=key, params=params), lambda ids: catalog.invocable_ids() <= ids, seconds=RELOAD_SECONDS * 4, ) return from_stem(marker, listed) def mine(plane: ControlPlane, credential_id: str) -> tuple[ControlPlaneRequest, ...]: return tuple(request for request in plane.drain() if request.credential == credential_id) def listings(requests: tuple[ControlPlaneRequest, ...]) -> Mapping[str, int]: return Counter(request.path for request in requests) def assert_listing_shape(requests: tuple[ControlPlaneRequest, ...], *, pages: int = 1) -> int: """Every listing is one foundation-models GET plus `pages` inference-profiles GETs; returns the listing count.""" assert requests, "no listing reached the control plane" counts: Final = listings(requests) assert set(counts) == {FOUNDATION_MODELS, INFERENCE_PROFILES}, counts assert 1 <= counts[FOUNDATION_MODELS] <= WORKERS, counts assert counts[INFERENCE_PROFILES] == counts[FOUNDATION_MODELS] * pages, counts for request in requests: assert request.method == "GET" and request.body == b"", request if request.path == FOUNDATION_MODELS: assert request.query == FOUNDATION_QUERY, request else: assert {name: value for name, value in request.query.items() if name != "nextToken"} == PROFILES_QUERY return counts[FOUNDATION_MODELS] def assert_sigv4(requests: tuple[ControlPlaneRequest, ...], *, key: str, secret: str, region: str) -> None: host: Final = control_plane_host(region) for request in requests: assert request.host == host, request assert request.headers["host"] == host, request scope: Final = verified_scope(request, secret) assert scope == f"{request.headers['x-amz-date'][:8]}/{region}/bedrock/aws4_request", request assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={key}/"), request assert "x-amz-security-token" not in request.headers, request def worker_pids(log: Path) -> tuple[int, ...]: return tuple(int(pid) for pid in STARTED_WORKER.findall(log.read_text())) def live_worker_pids(log: Path) -> tuple[int, ...]: return tuple(pid for pid in worker_pids(log) if psutil.pid_exists(pid)) def wait_for_every_worker(log: Path, workers: int) -> None: """Readiness answers from the first worker up; uvicorn replaces a child that dies at boot, so the rest can lag.""" def every_worker_serving() -> bool: return len(live_worker_pids(log)) >= workers and log.read_text().count(STARTUP_COMPLETE) >= workers eventually(every_worker_serving, bool, seconds=150) def wait_for_replacement_worker(log: Path, original: tuple[int, ...]) -> None: def replacement_is_serving(pids: tuple[int, ...]) -> bool: return len(pids) > len(original) and log.read_text().count(STARTUP_COMPLETE) > len(original) eventually(lambda: worker_pids(log), replacement_is_serving, seconds=150) def open_connections_to(pid: int, url: str) -> int: port: Final = urlsplit(url).port return sum( 1 for connection in psutil.Process(pid).net_connections(kind="tcp") if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port )