From c447c3312db1b99e058062f8cd61f2904fa1e9ef Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 16 Sep 2026 08:13:04 -0700 Subject: [PATCH] feat(e2e): separate a provider error from a body that failed its rule Build 226's 62 Bedrock rejections are the question this is trying to answer, and "incomplete" would have covered both candidate causes at once. Replaying the completeness rules over eight streams captured from live Bedrock, covering tool use, extended thinking and a max-tokens stop on both streaming endpoints, accepts every one of them, so a rule that is too strict is the less likely half. A provider that answered 429 or 5xx and was retried out of sight is the other, and it now counts as rejected_error_status rather than being folded in with a grammar failure. --- .../code_coverage_tests/test_provider_cache.py | 18 +++++++++++++++--- tests/e2e/PROVIDER_CACHE.md | 2 +- tests/e2e/provider_cache.py | 3 +++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/code_coverage_tests/test_provider_cache.py b/tests/code_coverage_tests/test_provider_cache.py index 1271b20438a..9668de35a02 100644 --- a/tests/code_coverage_tests/test_provider_cache.py +++ b/tests/code_coverage_tests/test_provider_cache.py @@ -546,13 +546,25 @@ def test_a_rejection_says_whether_the_body_was_cut_short_or_simply_unfinished( finally: second.shutdown() + refused: Final = cache_edge(store) + provider.status = 429 + provider.response = b'{"message":"Too many requests"}' + third: Final = start_provider_edge(refused, mounts={"openai": upstream}) + try: + call(third.edge.api_base("openai") + "/v1/chat/completions", MARKED) + finally: + third.shutdown() + cut: Final = dict(cut_short.counters.counts) turned_down: Final = dict(unfinished.counters.counts) - assert cut["mount:openai:rejected"] == 1 and turned_down["mount:openai:rejected"] == 1 + errored: Final = dict(refused.counters.counts) + assert cut["mount:openai:rejected"] == turned_down["mount:openai:rejected"] == errored["mount:openai:rejected"] == 1 assert cut["mount:openai:rejected_cut_short"] == 1 - assert "mount:openai:rejected_incomplete" not in cut assert turned_down["mount:openai:rejected_incomplete"] == 1 - assert "mount:openai:rejected_cut_short" not in turned_down + assert errored["mount:openai:rejected_error_status"] == 1 + assert not {"mount:openai:rejected_incomplete", "mount:openai:rejected_error_status"} & set(cut) + assert not {"mount:openai:rejected_cut_short", "mount:openai:rejected_error_status"} & set(turned_down) + assert not {"mount:openai:rejected_cut_short", "mount:openai:rejected_incomplete"} & set(errored) EMBEDDING_SUCCESS: Final = ( diff --git a/tests/e2e/PROVIDER_CACHE.md b/tests/e2e/PROVIDER_CACHE.md index 894d9be4efa..ed1e7d517b0 100644 --- a/tests/e2e/PROVIDER_CACHE.md +++ b/tests/e2e/PROVIDER_CACHE.md @@ -54,7 +54,7 @@ The trusted runner receives: - `E2E_PROVIDER_CACHE_NAMESPACE`: shared environment namespace, independent of build and candidate revision - `E2E_PROVIDER_CACHE_METRICS_DIR`: optional per-process counter artifact directory -Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. A rejection also counts its reason, one of `rejected_cut_short` (the consumer walked away mid-capture), `rejected_incomplete` (the body arrived whole and failed its endpoint's rule) or `rejected_unreachable` (the provider could not be reached). A mount whose rejections are nearly all one or the other is a different problem, and the flat count cannot tell them apart. Every counter is emitted twice, once as a flat total and once under `mount:{mount}:`, so a hit rate can be read per provider rather than only in aggregate. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits +Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. A rejection also counts its reason, one of `rejected_cut_short` (the consumer walked away mid-capture), `rejected_error_status` (the provider answered, with an error), `rejected_incomplete` (the body arrived whole with a success status and failed its endpoint's rule) or `rejected_unreachable` (the provider could not be reached at all). A mount whose rejections are nearly all one or the other is a different problem, and the flat count cannot tell them apart. Every counter is emitted twice, once as a flat total and once under `mount:{mount}:`, so a hit rate can be read per provider rather than only in aggregate. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read. diff --git a/tests/e2e/provider_cache.py b/tests/e2e/provider_cache.py index f528d08b720..399a0379889 100644 --- a/tests/e2e/provider_cache.py +++ b/tests/e2e/provider_cache.py @@ -55,6 +55,7 @@ EVENTSTREAM_PRELUDE_BYTES: Final = 4 CUT_SHORT: Final = "cut_short" INCOMPLETE: Final = "incomplete" UNREACHABLE: Final = "unreachable" +ERROR_STATUS: Final = "error_status" EVENT_TYPE_HEADER: Final = ":event-type" EVENTSTREAM_HEADERS: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) OPENAI_JSON_PATHS: Final = frozenset({"/v1/chat/completions", "/v1/messages", "/v1/embeddings", "/v1/responses"}) @@ -610,6 +611,8 @@ class CacheEdge: headers: Final = { name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS } + if not 200 <= head.status_code < 300: + return ERROR_STATUS chunks: Final = capture.chunks() if not successful_response(mount, url, head.status_code, headers, b"".join(chunks)): return INCOMPLETE