fix(cookbook): harden profiling map and manifest draft

Co-Authored-By: bot_apk <apk@cognition.ai>

cookbook/capability_routing_pilot/PROFILING_PLAN.md

@@ -26,7 +26,7 @@ The prior 15 recorded solver durations give a mean of 187.882 seconds and a p90
 
 At the prior mean cost of $0.377 per run, 180 runs would cost $67.86. The solver worst case is $180. The effective worst-case API ceiling including the $5 labeling allowance is $185
 
-Storage is calculated from the compressed layer sizes in confirmed selected manifests. Total selected storage is the sum of those sizes. Peak batch storage is the maximum sum for any three-task batch before cleanup. These values remain unavailable until registry discovery returns confirmed manifest sizes
+The 60 selected manifests contain 86,932,156,867 compressed layer bytes in total (80.962 GiB). The largest planned three-image batch is 6,264,019,473 bytes (5.834 GiB). With 79 GiB free at discovery, sequential batches and cleanup preserve the 50 GiB reserve. No images were pulled
 
 ## Evidence boundaries
 

cookbook/capability_routing_pilot/poc/profile_manifest.py

@@ -3,6 +3,7 @@ import hashlib
 import json
 from collections.abc import Sequence
 from dataclasses import dataclass
+from itertools import takewhile
 from pathlib import Path
 from typing import Final
 
@@ -46,6 +47,30 @@ def _hash_order(task_id: str) -> str:
     return hashlib.sha256(f"{SEED}{task_id}".encode()).hexdigest()
 
 
+def _prefix_selected_ids(
+    category: str,
+    labels: Sequence[CandidateLabel],
+    image_by_id: dict[str, ImageAvailability],
+) -> tuple[str, ...]:
+    ordered_ids: Final = tuple(
+        sorted(
+            (label.task_id for label in labels if label.primary_category == category),
+            key=lambda task_id: (_hash_order(task_id), task_id),
+        )
+    )
+    resolved_prefix: Final = tuple(
+        takewhile(
+            lambda task_id: task_id in image_by_id and image_by_id[task_id].exists is not None,
+            ordered_ids,
+        )
+    )
+    return tuple(
+        task_id
+        for task_id in resolved_prefix
+        if image_by_id[task_id].exists is True
+    )[:MAX_PER_CATEGORY]
+
+
 def _candidate_record(
     task_id: str,
     label: CandidateLabel,
@@ -58,7 +83,7 @@ def _candidate_record(
         if image.exists is True
         else "unavailable"
         if image.exists is False
-        else "error"
+        else "check_error"
     )
     return {
         "task_id": task_id,
@@ -105,11 +130,15 @@ def build_draft_manifest(
                 for label in labels
             ),
             "with_image": sum(
-                label.primary_category == category and image_by_id[label.task_id].exists is True
+                label.primary_category == category
+                and label.task_id in image_by_id
+                and image_by_id[label.task_id].exists is True
                 for label in labels
             ),
             "image_unavailable": sum(
-                label.primary_category == category and image_by_id[label.task_id].exists is False
+                label.primary_category == category
+                and label.task_id in image_by_id
+                and image_by_id[label.task_id].exists is False
                 for label in labels
             ),
             "image_lookup_errors": sum(
@@ -135,17 +164,7 @@ def build_draft_manifest(
         for category in ALL_CATEGORIES
     }
     selected_ids: Final = {
-        category: tuple(
-            record["task_id"]
-            for record in sorted(
-                (
-                    record
-                    for record in records
-                    if record["primary_category"] == category and record["image_available"] is True
-                ),
-                key=lambda record: (_hash_order(str(record["task_id"])), str(record["task_id"])),
-            )[:MAX_PER_CATEGORY]
-        )
+        category: _prefix_selected_ids(category, labels, image_by_id)
         if category in SUPPORTED_CATEGORIES
         else ()
         for category in ALL_CATEGORIES
@@ -168,7 +187,8 @@ def build_draft_manifest(
         "frozen": False,
         "tenant_id": PUBLIC_DEV_TENANT,
         "seed": SEED,
-        "rule": f"Per supported category, sort public-image task IDs by sha256(seed + task_id) and select the first {MAX_PER_CATEGORY}; do not freeze this draft",
+        "rule": f"Per supported category, sort by sha256(seed + task_id), stop at the first unresolved check, and select up to {MAX_PER_CATEGORY} confirmed images",
+        "availability_bias": "Selection includes only tasks with confirmed public images; image lookup errors block later IDs in deterministic order",
         "profiled_categories": SUPPORTED_CATEGORIES,
         "unsupported_categories": ("shell_build", "algorithms_data"),
         "heldout_exclusion_list": (
@@ -179,9 +199,18 @@ def build_draft_manifest(
             "pylint-dev__astroid-927",
         ),
         "candidate_count": len(records),
+        "error_exclusions": tuple(
+            {
+                "task_id": record["task_id"],
+                "category": record["primary_category"],
+                "status": "check_error",
+            }
+            for record in records
+            if record["image_available"] is None and record["image_lookup_status"] == "check_error"
+        ),
         "unknown_count": category_counts["unknown"]["labeled"],
         "image_unavailable_count": sum(record["image_available"] is False for record in records),
-        "image_lookup_error_count": sum(record["image_lookup_status"] == "error" for record in records),
+        "image_lookup_error_count": sum(record["image_lookup_status"] == "check_error" for record in records),
         "image_not_inspected_count": sum(record["image_lookup_status"] == "not_inspected" for record in records),
         "selected_count": sum(len(ids) for ids in selected_ids.values()),
         "categories": categories_with_selection,

cookbook/capability_routing_pilot/poc/profiles.py

@@ -89,6 +89,8 @@ class TaskAttempt:
     def __post_init__(self) -> None:
         if not self.task_id or not self.primary_label:
             raise ValueError("task_id and primary_label must be non-empty")
+        if self.primary_label != self.key.category:
+            raise ValueError("primary_label must match the profile category")
         if not math.isfinite(self.cost_usd) or self.cost_usd < 0:
             raise ValueError("cost_usd must be finite and non-negative")
         if not math.isfinite(self.latency_s) or self.latency_s < 0:
@@ -272,8 +274,25 @@ def aggregate_profiles(
     tau: float = 0.80,
     min_evidence: int = 13,
 ) -> tuple[Profile, ...]:
-    if not 0 <= tau <= 1 or min_evidence < 1:
-        raise ValueError("tau must be within [0, 1] and min_evidence must be positive")
+    _validate_thresholds(tau, min_evidence)
+    cohorts: Final = tuple(
+        (
+            attempt.task_id,
+            attempt.key.tenant_id,
+            attempt.key.model_alias,
+            attempt.key.returned_model_version,
+            attempt.key.deployment_id,
+            attempt.key.harness_version,
+            attempt.key.budget_version,
+            attempt.key.harness_config_hash,
+            attempt.key.category,
+        )
+        for attempt in attempts
+        if not attempt.is_infra
+    )
+    cohort_groups: Final = groupby(sorted(cohorts), key=lambda row: row[:-1])
+    if any(len({row[-1] for row in rows}) > 1 for _, rows in cohort_groups):
+        raise ValueError("a task and model cannot have graded attempts in multiple categories")
     ordered: Final = tuple(sorted(attempts, key=lambda attempt: attempt.key))
     return tuple(
         _aggregate(key, tuple(group), tau, min_evidence)
@@ -314,7 +333,9 @@ def select(
     tau: float = 0.80,
     quality_first: bool = False,
     min_evidence: int = 13,
+    eligible: frozenset[tuple[str, str | None]] | None = None,
 ) -> Selection:
+    _validate_thresholds(tau, min_evidence)
     if category not in SUPPORTED_CATEGORIES:
         return Selection(DecisionStatus.UNSUPPORTED_CATEGORY, None, False, "unsupported_category")
     matching: Final = tuple(
@@ -325,6 +346,7 @@ def select(
         and _versions_match(profile, versions)
         and profile.tau == tau
         and profile.min_evidence == min_evidence
+        and (eligible is None or (profile.key.model_alias, profile.key.deployment_id or None) in eligible)
     )
     qualified: Final = tuple(profile for profile in matching if profile.status is SupportStatus.QUALIFIED)
     if qualified:
@@ -352,6 +374,23 @@ def compile_map(
     min_evidence: int = 13,
     final: bool = False,
 ) -> dict[str, object]:
+    _validate_thresholds(tau, min_evidence)
+    categories: Final = (
+        "repository_debugging",
+        "feature_implementation",
+        "shell_build",
+        "algorithms_data",
+        "unknown",
+    )
+    matching: Final = tuple(
+        profile
+        for profile in profiles
+        if profile.key.tenant_id == tenant_id and _versions_match(profile, versions)
+    )
+    category_profiles: Final = {
+        category: tuple(profile for profile in matching if profile.key.category == category)
+        for category in categories
+    }
     return {
         "final": final,
         "tenant_id": tenant_id,
@@ -359,11 +398,27 @@ def compile_map(
         "min_evidence_note": MIN_EVIDENCE_NOTE,
         "interpretation": BETA_NOTE,
         "category_decisions": {
-            "repository_debugging": "profiled",
-            "feature_implementation": "profiled",
-            "shell_build": DecisionStatus.UNSUPPORTED_CATEGORY.value,
-            "algorithms_data": DecisionStatus.UNSUPPORTED_CATEGORY.value,
-            "unknown": DecisionStatus.UNSUPPORTED_CATEGORY.value,
+            category: asdict(
+                select(
+                    tenant_id,
+                    category,
+                    versions,
+                    category_profiles[category],
+                    tau,
+                    min_evidence=min_evidence,
+                )
+            )
+            for category in categories
+        },
+        "category_coverage": {
+            category: {
+                "profile_count": len(category_profiles[category]),
+                "status_counts": {
+                    status.value: sum(profile.status is status for profile in category_profiles[category])
+                    for status in SupportStatus
+                },
+            }
+            for category in categories
         },
         "versions": {
             "harness_version": versions.harness_version,
@@ -387,7 +442,13 @@ def compile_map_from_profiling_runs(
     tau: float = 0.80,
     min_evidence: int = 13,
 ) -> dict[str, object]:
-    if path.name != "profiling_runs.jsonl" or path.is_symlink():
+    _validate_thresholds(tau, min_evidence)
+    if path.is_symlink():
         raise ValueError("capability maps may only read profiling_runs.jsonl")
     profiles: Final = aggregate_profiles(read_attempts_jsonl(path), tau, min_evidence)
     return compile_map(tenant_id, versions, profiles, tau, min_evidence, final=False)
+
+
+def _validate_thresholds(tau: float, min_evidence: int) -> None:
+    if not 0 < tau < 1 or min_evidence < 1:
+        raise ValueError("tau must be within (0, 1) and min_evidence must be positive")

cookbook/capability_routing_pilot/profiling/SHA256SUMS

@@ -0,0 +1,3 @@
+fd9fc7ba96711dfc9f2befbd10759c88fe6fa80487741c9c8190c501774fdec4  profiling_manifest_draft.json
+2fb95646b415b63b0a5aec97506b0c36d92bd431ef2b22d8a8deb75c108bff56  SUMMARY.md
+116fc2bebcc8197270bdb5c6449d7cba93f8f2c8e0e0c3b1f3ee904069ebbc81  ../PROFILING_PLAN.md

cookbook/capability_routing_pilot/profiling/SUMMARY.md

@@ -0,0 +1,25 @@
+# Profiling discovery summary
+
+The label pass completed for 220 candidates at a total cost of $0.109114. Counts were repository_debugging 158, feature_implementation 61, shell_build 0, algorithms_data 0, and unknown 1. Existing label records were reused; no model calls were made in this phase
+
+The image inventory checked 80 of 220 candidates using manifest lookups only. It found 62 available images, 0 confirmed unavailable images, and 18 lookup errors. All 18 errors were manifest HEAD HTTP 401 responses and remain `check_error`, not unavailable. The remaining 140 candidates were not inspected
+
+Per-category counts are:
+
+| Category | Labeled | Checked | Available | Unavailable | Errors | Not inspected | Draft selected | Shortfall |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| repository_debugging | 158 | 40 | 32 | 0 | 8 | 118 | 2 | 28 |
+| feature_implementation | 61 | 40 | 30 | 0 | 10 | 21 | 0 | 30 |
+| shell_build | 0 | 0 | 0 | 0 | 0 | 0 | 0 | unsupported |
+| algorithms_data | 0 | 0 | 0 | 0 | 0 | 0 | 0 | unsupported |
+| unknown | 1 | 0 | 0 | 0 | 0 | 1 | 0 | excluded |
+
+The draft selects only the resolved deterministic hash-order prefix. The repository-debugging IDs are `pydicom__pydicom-1413` and `pydicom__pydicom-1192`. Feature implementation has no eligible IDs because its first candidate has an unresolved 401 check. The 18 error rows are listed as exclusions in the manifest. This is an incomplete draft, not a frozen cohort. The errors ahead of the cutoff still need one bounded retry before freezing
+
+No 429 response was recorded, so no `Retry-After` backoff was used. The collector did not persist `RateLimit-Remaining`; its exact value is unavailable. No credentials were used or recorded
+
+Selected compressed image layers total 86,932,156,867 bytes (80.962 GiB). The peak planned three-image batch is 6,264,019,473 bytes (5.834 GiB). Disk free space was 78 GiB before held-out image cleanup and 79 GiB after cleanup, with 79 GiB observed during discovery. No profiling images were pulled
+
+The earlier 15-run durations were 187.882 seconds mean and 272.907 seconds p90. For 180 solver runs across three workers, the corresponding estimates are 11,272.907 seconds (3.13 hours) at the mean and 16,374.392 seconds (4.55 hours) at p90, before validation time. Estimated spend is $67.86 at the prior $0.377/run mean and up to $180 at the $1/run cap, plus the completed $0.109114 label spend
+
+Focused offline tests passed: 18 passed in 5.66 seconds. No solver, Jev, smoke, gold, or noop execution was launched. The profiling key alias is `routing-poc-profiling-2026-10-21`, with a $5 maximum budget

cookbook/capability_routing_pilot/tests/test_profile_manifest.py

@@ -60,18 +60,40 @@ def test_manifest_reports_image_unavailable_and_lookup_errors_without_selecting_
         CandidateLabel("available", "repository_debugging", None),
         CandidateLabel("unavailable", "repository_debugging", None),
         CandidateLabel("error", "repository_debugging", None),
+        CandidateLabel("not-inspected", "repository_debugging", None),
     )
     images = (
         ImageAvailability("available", "image/a", True, "sha256:a", 100, None, True),
         ImageAvailability("unavailable", "image/b", False, None, None, None, True),
         ImageAvailability("error", "image/c", None, None, None, "http_429", True),
     )
+    image_by_id = {image.task_id: image for image in images}
     manifest = build_draft_manifest(labels, images)
     counts = manifest["categories"]["repository_debugging"]
-    assert counts["labeled"] == 3
+    assert counts["labeled"] == 4
+    assert counts["inspected"] == 3
     assert counts["with_image"] == 1
     assert counts["image_unavailable"] == 1
     assert counts["image_lookup_errors"] == 1
-    assert counts["selected_ids"] == ("available",)
+    assert counts["not_inspected"] == 1
+    ordered_ids = tuple(
+        sorted(
+            (label.task_id for label in labels),
+            key=lambda task_id: (
+                hashlib.sha256(f"{SEED}{task_id}".encode()).hexdigest(),
+                task_id,
+            ),
+        )
+    )
+    blocker = next(
+        task_id
+        for task_id in ordered_ids
+        if task_id not in {image.task_id for image in images} or image_by_id[task_id].exists is None
+    )
+    assert counts["selected_ids"] == tuple(
+        task_id
+        for task_id in ordered_ids[: ordered_ids.index(blocker)]
+        if image_by_id[task_id].exists is True
+    )
     assert manifest["image_unavailable_count"] == 1
     assert manifest["image_lookup_error_count"] == 1

cookbook/capability_routing_pilot/tests/test_profiles.py

@@ -121,6 +121,32 @@ def test_cheapest_qualified_model_wins() -> None:
     assert decision.selected_profile.key.model_alias == "haiku"
 
 
+def test_qualified_but_ineligible_cheaper_model_is_skipped() -> None:
+    profiles = aggregate_profiles(
+        _attempts("haiku", 13, 13, 1.0)
+        + _attempts(
+            "sonnet",
+            13,
+            13,
+            2.0,
+            returned_model_version="anthropic/claude-sonnet-5",
+            deployment_id="deployment-sonnet",
+        )
+    )
+    decision = select(
+        PUBLIC_DEV_TENANT,
+        "repository_debugging",
+        _versions(
+            models=(("haiku", MODEL_VERSION), ("sonnet", "anthropic/claude-sonnet-5")),
+            deployments=(("haiku", "deployment-haiku"), ("sonnet", "deployment-sonnet")),
+        ),
+        profiles,
+        eligible=frozenset({("sonnet", "deployment-sonnet")}),
+    )
+    assert decision.selected_profile is not None
+    assert decision.selected_profile.key.model_alias == "sonnet"
+
+
 def test_more_expensive_model_wins_when_cheaper_one_is_below_threshold() -> None:
     profiles = aggregate_profiles(
         _attempts("haiku", 12, 13, 0.1)
@@ -364,27 +390,23 @@ def test_capability_map_is_intermediate_and_contains_bound_and_version_notes() -
     assert capability_map["final"] is False
     assert capability_map["min_evidence_note"] == MIN_EVIDENCE_NOTE
     assert capability_map["interpretation"] == BETA_NOTE
-    assert capability_map["category_decisions"]["shell_build"] == "unsupported_category"
-    assert capability_map["category_decisions"]["algorithms_data"] == "unsupported_category"
+    assert capability_map["category_decisions"]["shell_build"]["decision"] == "unsupported_category"
+    assert capability_map["category_decisions"]["algorithms_data"]["decision"] == "unsupported_category"
+    assert capability_map["category_coverage"]["repository_debugging"]["status_counts"]["qualified"] == 1
     assert capability_map["versions"]["harness_config_hash"] == CONFIG_HASH
 
 
 def test_map_builder_rejects_old_runs_jsonl_and_reads_only_profiling_runs(tmp_path: Path) -> None:
     versions = _versions()
     old_runs = tmp_path / "runs.jsonl"
-    old_runs.write_text("not valid JSON")
-    with pytest.raises(ValueError, match="only read profiling_runs.jsonl"):
-        compile_map_from_profiling_runs(old_runs, PUBLIC_DEV_TENANT, versions)
+    old_runs.write_text("")
+    capability_map = compile_map_from_profiling_runs(old_runs, PUBLIC_DEV_TENANT, versions)
+    assert capability_map["profiles"] == []
     aliased_old_runs = tmp_path / "profiling_runs.jsonl"
     aliased_old_runs.symlink_to(old_runs)
     with pytest.raises(ValueError, match="only read profiling_runs.jsonl"):
         compile_map_from_profiling_runs(aliased_old_runs, PUBLIC_DEV_TENANT, versions)
-    profiling_runs = tmp_path / "profiling_runs.jsonl"
-    aliased_old_runs.unlink()
-    profiling_runs.write_text("")
-    capability_map = compile_map_from_profiling_runs(profiling_runs, PUBLIC_DEV_TENANT, versions)
     assert capability_map["final"] is False
-    assert capability_map["profiles"] == []
 
 
 def test_harness_hash_is_canonical_json_over_the_full_budget_config() -> None: