feat(e2e): separate a provider error from a body that failed its rule

Build 226's 62 Bedrock rejections are the question this is trying to
answer, and "incomplete" would have covered both candidate causes at once.
Replaying the completeness rules over eight streams captured from live
Bedrock, covering tool use, extended thinking and a max-tokens stop on
both streaming endpoints, accepts every one of them, so a rule that is too
strict is the less likely half. A provider that answered 429 or 5xx and
was retried out of sight is the other, and it now counts as
rejected_error_status rather than being folded in with a grammar failure.

tests/code_coverage_tests/test_provider_cache.py

@@ -546,13 +546,25 @@ def test_a_rejection_says_whether_the_body_was_cut_short_or_simply_unfinished(
     finally:
         second.shutdown()
 
+    refused: Final = cache_edge(store)
+    provider.status = 429
+    provider.response = b'{"message":"Too many requests"}'
+    third: Final = start_provider_edge(refused, mounts={"openai": upstream})
+    try:
+        call(third.edge.api_base("openai") + "/v1/chat/completions", MARKED)
+    finally:
+        third.shutdown()
+
     cut: Final = dict(cut_short.counters.counts)
     turned_down: Final = dict(unfinished.counters.counts)
-    assert cut["mount:openai:rejected"] == 1 and turned_down["mount:openai:rejected"] == 1
+    errored: Final = dict(refused.counters.counts)
+    assert cut["mount:openai:rejected"] == turned_down["mount:openai:rejected"] == errored["mount:openai:rejected"] == 1
     assert cut["mount:openai:rejected_cut_short"] == 1
-    assert "mount:openai:rejected_incomplete" not in cut
     assert turned_down["mount:openai:rejected_incomplete"] == 1
-    assert "mount:openai:rejected_cut_short" not in turned_down
+    assert errored["mount:openai:rejected_error_status"] == 1
+    assert not {"mount:openai:rejected_incomplete", "mount:openai:rejected_error_status"} & set(cut)
+    assert not {"mount:openai:rejected_cut_short", "mount:openai:rejected_error_status"} & set(turned_down)
+    assert not {"mount:openai:rejected_cut_short", "mount:openai:rejected_incomplete"} & set(errored)
 
 
 EMBEDDING_SUCCESS: Final = (

tests/e2e/PROVIDER_CACHE.md

@@ -54,7 +54,7 @@ The trusted runner receives:
 - `E2E_PROVIDER_CACHE_NAMESPACE`: shared environment namespace, independent of build and candidate revision
 - `E2E_PROVIDER_CACHE_METRICS_DIR`: optional per-process counter artifact directory
 
-Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. A rejection also counts its reason, one of `rejected_cut_short` (the consumer walked away mid-capture), `rejected_incomplete` (the body arrived whole and failed its endpoint's rule) or `rejected_unreachable` (the provider could not be reached). A mount whose rejections are nearly all one or the other is a different problem, and the flat count cannot tell them apart. Every counter is emitted twice, once as a flat total and once under `mount:{mount}:`, so a hit rate can be read per provider rather than only in aggregate. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits
+Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. A rejection also counts its reason, one of `rejected_cut_short` (the consumer walked away mid-capture), `rejected_error_status` (the provider answered, with an error), `rejected_incomplete` (the body arrived whole with a success status and failed its endpoint's rule) or `rejected_unreachable` (the provider could not be reached at all). A mount whose rejections are nearly all one or the other is a different problem, and the flat count cannot tell them apart. Every counter is emitted twice, once as a flat total and once under `mount:{mount}:`, so a hit rate can be read per provider rather than only in aggregate. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits
 
 Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read.
 

tests/e2e/provider_cache.py

@@ -55,6 +55,7 @@ EVENTSTREAM_PRELUDE_BYTES: Final = 4
 CUT_SHORT: Final = "cut_short"
 INCOMPLETE: Final = "incomplete"
 UNREACHABLE: Final = "unreachable"
+ERROR_STATUS: Final = "error_status"
 EVENT_TYPE_HEADER: Final = ":event-type"
 EVENTSTREAM_HEADERS: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 OPENAI_JSON_PATHS: Final = frozenset({"/v1/chat/completions", "/v1/messages", "/v1/embeddings", "/v1/responses"})
@@ -610,6 +611,8 @@ class CacheEdge:
         headers: Final = {
             name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS
         }
+        if not 200 <= head.status_code < 300:
+            return ERROR_STATUS
         chunks: Final = capture.chunks()
         if not successful_response(mount, url, head.status_code, headers, b"".join(chunks)):
             return INCOMPLETE