"""Provider-edge record/replay server for e2e runs (LIT-5745). Record and replay scope to provider-bound traffic only: the proxy boots for real, tests hit it for real, and only the hop from the proxy to the provider is recorded or served from a bundle. Suites opt in per deployment by pointing ``litellm_params.api_base`` at ``provider_edge_api_base(mount)``, which is an in-process HTTP server mounting each supported provider under a path prefix (``http://127.0.0.1:/openai`` forwards to ``https://api.openai.com``). In record mode the edge relays each request verbatim, stores the interaction, and serves the proxy the same filtered response replay will serve later; in replay mode it serves straight from the bundle and never opens a provider connection, so a green replay run with a fake provider key proves the entire proxy pipeline (auth, routing, spend logging) without provider spend. Request identity reuses fixture_canonical.py: interactions match by canonical content key, order-independent across keys and FIFO within one. Edge requests store no headers at all: SDK telemetry headers vary run to run and credential headers must never touch disk. An unmatched replay call returns HTTP ``REPLAY_MISS_STATUS`` naming the closest recorded interaction, which the proxy relays as a provider error the failing test surfaces. A response the provider streamed (one whose content type names ``text/event-stream``) is relayed and stored chunk by chunk instead of buffered (LIT-5742): the edge reads one piece per upstream transfer chunk, writes each one downstream in chunked framing as it arrives, and records the sequence, so replay hands the proxy the same number of chunks split in the same places. A provider that hangs up mid-stream is recorded as the chunks it did deliver plus a truncation, and replays as those chunks followed by a connection close with no terminator, which is the same incomplete chunked read the live failure produced rather than a clean 502 that erases it. Everything else keeps the buffered shape, byte for byte, framed with a content-length as before. v1 limits: only the mounts in ``EDGE_MOUNTS`` (SigV4 providers like Bedrock sign the Host header, so a forwarding edge breaks their signatures), and CI wiring is LIT-5748. Suites that do not wire the edge keep hitting providers live in every mode. """ from __future__ import annotations import base64 import difflib import functools import hashlib import re import threading from collections import deque from collections.abc import Mapping, Sequence from contextlib import closing from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from itertools import islice from pathlib import Path from types import MappingProxyType from typing import Final, Generator, Literal, assert_never from urllib.parse import parse_qsl, urlsplit from pydantic import JsonValue, TypeAdapter from e2e_http import ( NetworkError, StreamChunk, StreamHead, StreamStep, StreamTruncation, forward_stream, ) from fixture_bundle import ( BundleRecorder, Interaction, LoadedBundle, RecordedHttpResponse, RecordedRequest, RecordedResponse, RecordedStreamedResponse, UnreadableBundle, UnsafeBundleDir, interaction_filename, load_bundle, prepare_bundle, slug_for_test, ) from fixture_canonical import ( SECRET_PLACEHOLDER, CanonicalRequest, canonical_string, canonicalize, is_secret_field, ) from fixture_mode import ( FIXTURE_MODES, InvalidFixtureMode, ReplayMiss, current_test_key, parse_fixture_mode, ) EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( { "openai": "https://api.openai.com", "anthropic": "https://api.anthropic.com", } ) REPLAY_MISS_STATUS: Final = 599 _HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( { "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailers", "transfer-encoding", "upgrade", } ) _REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { "host", "content-length", "accept-encoding", } _RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { "content-encoding", "content-length", "set-cookie", } _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _BOUNDARY_PATTERN: Final = re.compile( r'(?:^|;)\s*boundary\s*=\s*(?:"([^"]*)"|([^;,\s]+))', re.IGNORECASE ) _DISPOSITION_NAME_PATTERN: Final = re.compile(r'(?:^|;)\s*name="([^"]*)"', re.IGNORECASE) _DISPOSITION_FILENAME_PATTERN: Final = re.compile( r'(?:^|;)\s*filename="([^"]*)"', re.IGNORECASE ) _UNPARSED_MULTIPART: Final = "" _BOUNDARY_PLACEHOLDER: Final = b"--" _BINARY_FIELD_PREFIX: Final = " str: wanted: Final = name.lower() return next((value for key, value in headers.items() if key.lower() == wanted), "") def _multipart_boundary(content_type: str) -> str | None: """The declared boundary, or None when the envelope is not multipart or names no usable boundary. ``boundary`` is matched only as a parameter in its own right, so a longer name ending in it (``myboundary=``) is not mistaken for one, and an empty boundary is refused rather than splitting the body on a bare ``--``.""" if "multipart/form-data" not in content_type.lower(): return None match: Final = _BOUNDARY_PATTERN.search(content_type) if match is None: return None quoted, bare = match.group(1), match.group(2) return (quoted if quoted is not None else bare) or None def _part_headers(head: bytes) -> dict[str, str]: return { name.strip().lower(): value.strip() for line in head.decode("utf-8", errors="replace").split("\r\n") for name, separator, value in [line.partition(":")] if separator } def _parse_multipart_part(segment: bytes) -> _MultipartPart | None: head, separator, content = segment.partition(b"\r\n\r\n") if not separator: return None headers: Final = _part_headers(head) disposition: Final = headers.get("content-disposition", "") name_match: Final = _DISPOSITION_NAME_PATTERN.search(disposition) if name_match is None: return None filename_match: Final = _DISPOSITION_FILENAME_PATTERN.search(disposition) return _MultipartPart( field_name=name_match.group(1), filename=None if filename_match is None else filename_match.group(1), content=content, content_type=headers.get("content-type", ""), ) def _multipart_parts(body: bytes, boundary: str) -> tuple[_MultipartPart, ...] | None: """The wire body split back into its parts, or None when it does not parse as the declared envelope so the caller can fall back to the opaque content digest.""" segments: Final = body.split(b"--" + boundary.encode()) if len(segments) < 3 or not segments[-1].startswith(b"--"): return None parsed: Final = tuple( _parse_multipart_part(segment.removeprefix(b"\r\n").removesuffix(b"\r\n")) for segment in segments[1:-1] ) if any(part is None for part in parsed): return None return tuple(part for part in parsed if part is not None) def _content_digest(content: bytes) -> str: """Text is canonicalized before hashing so a per-run marker inside an uploaded JSONL does not move the key; anything that is not UTF-8 is hashed byte for byte, since a lossy decode collapses every binary payload of one length onto one digest.""" try: text: Final = content.decode("utf-8") except UnicodeDecodeError: return hashlib.sha256(content).hexdigest() return hashlib.sha256(canonical_string(text).encode()).hexdigest() def _is_file_part(part: _MultipartPart) -> bool: """Whether a part is an upload rather than an ordinary field. A filename says so outright, and so does a declared content type: clients attach one per part only for a file, and a client that omits the filename (httpx drops the parameter when it is empty) would otherwise have the file's bytes stored inline as a field value and key identically to a plain field of the same name.""" return part.filename is not None or bool(part.content_type) def _field_value(part: _MultipartPart) -> str: """What a field part contributes to the stored form. A secret-named field never has its value written out, since the bundle is a file on disk and the key redacts that field to the same placeholder either way, so replay still matches. A value that is not UTF-8 is carried as a digest rather than decoded lossily, because a replacing decode collapses every binary value of one length onto one string. That digest is base64 rather than hex, since the canonicalizer rewrites any long hex run to a ```` placeholder and would collapse the values right back together.""" if is_secret_field(part.field_name): return SECRET_PLACEHOLDER try: return part.content.decode("utf-8") except UnicodeDecodeError: digest: Final = base64.b64encode(hashlib.sha256(part.content).digest()).decode() return f"{_BINARY_FIELD_PREFIX}{digest}>" def _form_fields(fields: tuple[_MultipartPart, ...]) -> dict[str, str]: """The ordinary field parts, flattened into the mapping the bundle format stores. A name sent more than once takes an occurrence suffix instead of overwriting the earlier value, so nothing an upload said is dropped from its key. The suffix is escaped so a field literally named ``x[1]`` cannot collide with a second ``x``.""" form: dict[str, str] = {} for part in fields: name = part.field_name.replace("[", "[[") occurrence = 1 while name in form: name = f"{part.field_name.replace('[', '[[')}[{occurrence}]" occurrence += 1 form[name] = _field_value(part) return form def _file_identity(files: tuple[_MultipartPart, ...]) -> tuple[str | None, str | None, int | None]: """Name, content digest, and total length for the uploaded file parts. The name is a structured list of every part's field name, filename, and declared content type rather than a joined string, so a filename containing the separator cannot be confused for a different split, and two parts that differ only in the type they declare stay apart. It goes through the canonicalizer as one string, which is why per-run markers inside a filename do not move the key in the multi-file case any more than they do in the single-file one. The digest covers content only. A lone file keeps its own canonicalized digest; several fold into one ordered digest, so parts arriving in a different order key differently. Total length is recorded for a reader but deliberately kept out of the key: it is the raw byte count, and keying on it would undo exactly the drift the canonicalized digest exists to absorb.""" if not files: return None, None, None names: Final = _JSON.dump_json( [[part.field_name, part.filename, part.content_type] for part in files] ).decode() total: Final = sum(len(part.content) for part in files) if len(files) == 1: return names, _content_digest(files[0].content), total folded: Final = _JSON.dump_json([_content_digest(part.content) for part in files]) return names, hashlib.sha256(folded).hexdigest(), total def _multipart_request( method: str, path: str, params: dict[str, str], parts: tuple[_MultipartPart, ...] ) -> RecordedRequest: """A multipart upload keyed by what it says rather than by its wire bytes: every ordinary field, plus the identity of the uploaded file. The random per-request boundary is envelope, never content, so it never reaches the digest.""" form: Final = _form_fields(tuple(part for part in parts if not _is_file_part(part))) file_name, file_sha256, file_bytes = _file_identity( tuple(part for part in parts if _is_file_part(part)) ) return RecordedRequest( method=method, path=path, headers={}, params=params, form=form, file_name=file_name, file_sha256=file_sha256, file_bytes=file_bytes, ) def _opaque_request( method: str, path: str, params: dict[str, str], body: bytes, digested: bytes, file_name: str | None = None, ) -> RecordedRequest: """A body kept out of the bundle and matched on its digest alone. ``digested`` is what the digest runs over, which is the body itself unless something in it has to be normalized away first.""" return RecordedRequest( method=method, path=path, headers={}, params=params, file_name=file_name, file_sha256=_content_digest(digested), file_bytes=len(body), ) def edge_request( method: str, path: str, query: str, body: bytes | None, content_type: str = "" ) -> RecordedRequest: """The identity replay matches on: the edge path (mount included), the query as params, and the body as parsed JSON, as parsed multipart fields and file identity when the content type declares an envelope, or as a content digest otherwise so opaque uploads still match across runs. A multipart body that does not parse still has its boundary normalized away, because that boundary is fresh every request and would otherwise guarantee a miss.""" params: Final = dict(parse_qsl(query, keep_blank_values=True)) lowered_method: Final = method.lower() if not body: return RecordedRequest(method=lowered_method, path=path, headers={}, params=params) boundary: Final = _multipart_boundary(content_type) if boundary is not None: parts = _multipart_parts(body, boundary) if parts is not None: return _multipart_request(lowered_method, path, params, parts) return _opaque_request( lowered_method, path, params, body, body.replace(b"--" + boundary.encode(), _BOUNDARY_PLACEHOLDER), _UNPARSED_MULTIPART, ) try: parsed: Final[JsonValue] = _JSON.validate_json(body) except ValueError: return _opaque_request(lowered_method, path, params, body, body) return RecordedRequest( method=lowered_method, path=path, headers={}, params=params, body=parsed ) def _build_pool(recorded: tuple[Interaction, ...]) -> dict[str, deque[Interaction]]: keys: Final = tuple(canonicalize(interaction.request).key for interaction in recorded) return { key: deque( interaction for candidate_key, interaction in zip(keys, recorded, strict=True) if candidate_key == key ) for key in dict.fromkeys(keys) } def _closest_recorded( canonical: CanonicalRequest, recorded: tuple[Interaction, ...] ) -> tuple[CanonicalRequest, str]: candidates: Final = tuple(canonicalize(interaction.request) for interaction in recorded) ratios: Final = tuple( difflib.SequenceMatcher( None, f"{canonical.method} {canonical.path}\n{canonical.content}", f"{candidate.method} {candidate.path}\n{candidate.content}", ).ratio() for candidate in candidates ) best: Final = max(range(len(candidates)), key=lambda index: ratios[index]) return candidates[best], interaction_filename(best, recorded[best].request) def _miss_message(test_key: str, slug: str, canonical: CanonicalRequest, bundle: LoadedBundle) -> str: recorded: Final = bundle.interactions.get(slug, ()) if not recorded: return ( f"replay miss for {test_key}: computed key {canonical.key} but nothing is recorded " f"under {slug}; re-record with E2E_FIXTURE_MODE=record" ) closest, closest_file = _closest_recorded(canonical, recorded) diff: Final = "\n".join( islice( difflib.unified_diff( closest.pretty_content().splitlines(), canonical.pretty_content().splitlines(), fromfile=f"closest recorded ({closest_file})", tofile="test made", lineterm="", ), 60, ) ) return ( f"replay miss for {test_key}: no recorded interaction matches key {canonical.key}; " f"closest recorded key is {closest.key} ({closest_file})\n{diff}\n" "re-record with E2E_FIXTURE_MODE=record" ) @dataclass(slots=True) class ReplaySource: """One shared pool per test over a loaded bundle, so every provider call the proxy makes in the session consumes from the same recorded interactions. Every pool is built once at construction and per-key consumption is a single atomic deque pop, so concurrent replay calls never race. Calls match by canonical content key: order-independent across distinct keys (concurrent tests interleave calls nondeterministically), FIFO within one key (a retry or poll loop replays its recorded responses in recorded order).""" bundle: LoadedBundle _pools: dict[str, dict[str, deque[Interaction]]] = field(init=False) def __post_init__(self) -> None: self._pools = { slug: _build_pool(recorded) for slug, recorded in self.bundle.interactions.items() } def _pool(self, slug: str) -> dict[str, deque[Interaction]]: return self._pools.get(slug, {}) def next_interaction(self, request: RecordedRequest) -> Interaction: test_key: Final = current_test_key() slug: Final = slug_for_test(test_key) pool: Final = self._pool(slug) canonical: Final = canonicalize(request) queue: Final = pool.get(canonical.key) if queue is None: raise ReplayMiss(_miss_message(test_key, slug, canonical, self.bundle)) try: return queue.popleft() except IndexError: raise ReplayMiss( f"replay exhausted for {test_key}: every recorded interaction for key " f"{canonical.key} is already consumed; re-record with E2E_FIXTURE_MODE=record" ) from None def leftover_error(self, test_key: str) -> str | None: """Non-None when the test consumed fewer interactions than were recorded, meaning a passing replay proved less than the bundle claims.""" slug: Final = slug_for_test(test_key) recorded: Final = self.bundle.interactions.get(slug, ()) if not recorded: return None leftover: Final = tuple( interaction for queue in self._pool(slug).values() for interaction in queue ) if not leftover: return None return ( f"replay incomplete for {test_key}: {len(leftover)} of {len(recorded)} recorded " f"interactions never consumed, e.g. {canonicalize(leftover[0].request).key}; " "re-record with E2E_FIXTURE_MODE=record" ) @dataclass(frozen=True, slots=True) class RecordEdge: """Record backend: forward to the provider, persist, serve the filtered copy. The lock serializes recorder writes because the edge server handles requests on concurrent threads.""" recorder: BundleRecorder lock: threading.Lock @dataclass(frozen=True, slots=True) class ReplayEdge: source: ReplaySource type EdgeBackend = RecordEdge | ReplayEdge @dataclass(frozen=True, slots=True) class EdgeReply: """A whole response the edge already holds: written with a content-length.""" status_code: int headers: dict[str, str] body: bytes @dataclass(frozen=True, slots=True) class EdgeStream: """A response the edge relays chunk by chunk: written in chunked framing, one transfer chunk per step, so the split points reach the proxy intact. Record and replay both produce one of these, driven by different step sources, which is what makes their framing identical by construction rather than by inspection.""" status_code: int headers: dict[str, str] steps: Generator[StreamStep, None, None] type EdgeOutcome = EdgeReply | EdgeStream def _text_reply(status_code: int, message: str) -> EdgeReply: return EdgeReply( status_code=status_code, headers={"content-type": "text/plain; charset=utf-8"}, body=message.encode(), ) def _recorded_steps( chunks_b64: Sequence[str], truncated: str | None ) -> Generator[StreamStep, None, None]: """Replay's step source: the recorded chunks in recorded order, as fast as the socket takes them (inter-chunk delays are deliberately not reproduced), then the recorded truncation if the stream ended without a terminator.""" for chunk in chunks_b64: yield StreamChunk(data=base64.b64decode(chunk)) if truncated is not None: yield StreamTruncation(reason=truncated) def _recorded_outcome(response: RecordedResponse) -> EdgeOutcome: match response: case RecordedHttpResponse(status_code=status_code, headers=headers, body_b64=body_b64): return EdgeReply( status_code=status_code, headers=dict(headers), body=base64.b64decode(body_b64), ) case RecordedStreamedResponse( status_code=status_code, headers=headers, chunks_b64=chunks_b64, truncated=truncated ): return EdgeStream( status_code=status_code, headers=dict(headers), steps=_recorded_steps(chunks_b64, truncated), ) case _: assert_never(response) def _filtered_response_headers(headers: Mapping[str, str]) -> dict[str, str]: """What the edge stores and serves: the provider's headers minus hop-by-hop and volatile entries. Framing headers are in that set, so a stored header can never contradict the framing the edge chooses when it serves the response.""" return { name: value for name, value in headers.items() if name not in _RESPONSE_DROPPED_HEADERS } def _network_error_response(message: str) -> RecordedHttpResponse: return RecordedHttpResponse( status_code=502, headers={"content-type": "text/plain; charset=utf-8"}, body_b64=base64.b64encode( f"provider edge could not reach the provider: {message}".encode() ).decode("ascii"), ) def _buffered_response( status_code: int, headers: Mapping[str, str], body: bytes ) -> RecordedHttpResponse: return RecordedHttpResponse( status_code=status_code, headers=_filtered_response_headers(headers), body_b64=base64.b64encode(body).decode("ascii"), ) def _streamed_response( status_code: int, headers: Mapping[str, str], chunks: Sequence[bytes], truncated: str | None ) -> RecordedStreamedResponse: return RecordedStreamedResponse( status_code=status_code, headers=_filtered_response_headers(headers), chunks_b64=[base64.b64encode(chunk).decode("ascii") for chunk in chunks], truncated=truncated, ) def _is_streamed(headers: Mapping[str, str]) -> bool: """Whether a response is one to relay incrementally, decided by content type. ``transfer-encoding: chunked`` would be the wrong signal: chunking is a transport choice providers make freely for ordinary JSON, so keying off it would move nearly every recording to the streamed shape for no gain. The content type is the header that says "consume this as it arrives", and it is already how the harness defines streaming everywhere else.""" return "text/event-stream" in _header_value(headers, "content-type").lower() def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: url: Final = f"{upstream_base}/{upstream_path}" return f"{url}?{query}" if query else url def _persist( backend: RecordEdge, test_key: str, request: RecordedRequest, response: RecordedResponse ) -> None: with backend.lock: backend.recorder.record(test_key=test_key, request=request, response=response) def _recording_steps( backend: RecordEdge, test_key: str, request: RecordedRequest, head: StreamHead ) -> Generator[StreamStep, None, None]: """Record mode's step source: hand each upstream chunk downstream and record it only once that write has returned, then persist the whole sequence once, under the lock. Relaying incrementally keeps record exercising the proxy's incremental parser the way a live run does. A chunk is appended after its ``yield`` returns, so a downstream that hangs up mid-relay records exactly the chunks it took and never the one whose write raised. The ``except`` covers that downstream close and the proxy hanging up mid-stream; either way the ``finally`` persists what arrived, marked truncated, because recording a cut-short stream as a clean one would let a later replay serve a well-terminated fraction of the response and pass a test that should have gone red.""" collected: list[bytes] = [] truncated: str | None = None try: with closing(head.steps) as steps: for step in steps: match step: case StreamChunk(): pass case StreamTruncation(reason=reason): truncated = f"upstream: {reason}" case _: assert_never(step) yield step if isinstance(step, StreamChunk): collected.append(step.data) except GeneratorExit: if truncated is None: truncated = f"downstream: relay closed after {len(collected)} chunks" raise finally: _persist( backend, test_key, request, _streamed_response(head.status_code, head.headers, collected, truncated), ) def _drain_to_response(head: StreamHead) -> RecordedHttpResponse: """A response the detection rule did not call streamed: drain the same step iterator, join the pieces, and store today's buffered shape byte for byte. A truncation part way through degrades to the synthetic 502 exactly as the eager read did, because storing half a JSON body under a content-length as though it were whole would be a worse lie than failing.""" pieces: list[bytes] = [] with closing(head.steps) as steps: for step in steps: match step: case StreamChunk(data=data): pieces.append(data) case StreamTruncation(reason=reason): return _network_error_response(reason) case _: assert_never(step) return _buffered_response(head.status_code, head.headers, b"".join(pieces)) def _handle_record( backend: RecordEdge, request: RecordedRequest, *, method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float, ) -> EdgeOutcome: test_key: Final = current_test_key() forwarded: Final = { name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS } head: Final = forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) match head: case NetworkError(message=message): unreachable: Final = _network_error_response(message) _persist(backend, test_key, request, unreachable) return _recorded_outcome(unreachable) case StreamHead() if _is_streamed(head.headers): return EdgeStream( status_code=head.status_code, headers=_filtered_response_headers(head.headers), steps=_recording_steps(backend, test_key, request, head), ) case StreamHead(): buffered: Final = _drain_to_response(head) _persist(backend, test_key, request, buffered) return _recorded_outcome(buffered) case _: assert_never(head) def _handle_replay(source: ReplaySource, request: RecordedRequest) -> EdgeOutcome: try: interaction: Final = source.next_interaction(request) except ReplayMiss as miss: return _text_reply(REPLAY_MISS_STATUS, str(miss)) return _recorded_outcome(interaction.response) def handle_edge_request( backend: EdgeBackend, mounts: Mapping[str, str], method: str, raw_path: str, headers: Mapping[str, str], body: bytes | None, *, timeout: float, ) -> EdgeOutcome: """The edge's pure core, one HTTP exchange in and out: resolve the mount prefix, then record (forward + persist) or replay (serve from the bundle). Socket-free so unit tests exercise every branch without a server.""" split: Final = urlsplit(raw_path) mount, _, upstream_path = split.path.lstrip("/").partition("/") upstream_base: Final = mounts.get(mount) if upstream_base is None: return _text_reply( 404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}" ) request: Final = edge_request( method, split.path, split.query, body, _header_value(headers, "content-type") ) match backend: case RecordEdge(): return _handle_record( backend, request, method=method, url=_upstream_url(upstream_base, upstream_path, split.query), headers=headers, body=body, timeout=timeout, ) case ReplayEdge(source=source): return _handle_replay(source, request) case _: assert_never(backend) class _EdgeHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def do_GET(self) -> None: self._handle() def do_POST(self) -> None: self._handle() def do_PUT(self) -> None: self._handle() def do_PATCH(self) -> None: self._handle() def do_DELETE(self) -> None: self._handle() def _handle(self) -> None: edge_server: Final = self.server assert isinstance(edge_server, _EdgeHTTPServer) length: Final = int(self.headers.get("content-length") or "0") body: Final = self.rfile.read(length) if length else None outcome: Final = handle_edge_request( edge_server.backend, edge_server.mounts, self.command, self.path, {name.lower(): value for name, value in self.headers.items()}, body, timeout=edge_server.forward_timeout, ) match outcome: case EdgeReply(): self._write_reply(outcome) case EdgeStream(): self._write_stream(outcome) case _: assert_never(outcome) def _write_reply(self, reply: EdgeReply) -> None: self.send_response(reply.status_code) for name, value in reply.headers.items(): self.send_header(name, value) self.send_header("content-length", str(len(reply.body))) self.end_headers() self.wfile.write(reply.body) def _write_stream(self, stream: EdgeStream) -> None: """Write a streamed outcome in chunked framing, one transfer chunk per step. ``wbufsize`` is 0 on BaseHTTPRequestHandler, so ``wfile`` sends each write straight down the socket and no flush is needed. A truncation step ends the message without its terminator and closes the connection, which the stdlib shuts down write-side first: the proxy sees a graceful close mid-message, which is the incomplete chunked read a provider hanging up produces, and not the reset that could discard the chunks already in flight.""" self.send_response(stream.status_code) for name, value in stream.headers.items(): self.send_header(name, value) self.send_header("transfer-encoding", "chunked") self.end_headers() with closing(stream.steps) as steps: for step in steps: match step: case StreamChunk(data=data): self.wfile.write(b"%x\r\n%s\r\n" % (len(data), data)) case StreamTruncation(): self.close_connection = True return case _: assert_never(step) self.wfile.write(b"0\r\n\r\n") def log_message(self, format: str, *args: object) -> None: """Silence the per-request stderr line BaseHTTPRequestHandler emits.""" class _EdgeHTTPServer(ThreadingHTTPServer): daemon_threads = True def __init__( self, bind: tuple[str, int], *, backend: EdgeBackend, mounts: Mapping[str, str], forward_timeout: float, ) -> None: super().__init__(bind, _EdgeHandler) self.backend: Final = backend self.mounts: Final = mounts self.forward_timeout: Final = forward_timeout @dataclass(frozen=True, slots=True) class ProviderEdge: port: int advertise_host: str def api_base(self, mount: str) -> str: return f"http://{self.advertise_host}:{self.port}/{mount}" @dataclass(frozen=True, slots=True) class RunningEdge: edge: ProviderEdge server: _EdgeHTTPServer def shutdown(self) -> None: self.server.shutdown() self.server.server_close() def start_provider_edge( backend: EdgeBackend, *, mounts: Mapping[str, str] = EDGE_MOUNTS, bind_host: str = "127.0.0.1", advertise_host: str | None = None, forward_timeout: float = 60.0, ) -> RunningEdge: """Boot an edge server on an OS-assigned port in a daemon thread. ``advertise_host`` is what api_base URLs name (it differs from the bind host when the proxy runs in a container and reaches the host machine via a gateway address like host.docker.internal).""" server: Final = _EdgeHTTPServer( (bind_host, 0), backend=backend, mounts=mounts, forward_timeout=forward_timeout ) thread: Final = threading.Thread(target=server.serve_forever, name="e2e-provider-edge", daemon=True) thread.start() return RunningEdge( edge=ProviderEdge(port=server.server_address[1], advertise_host=advertise_host or bind_host), server=server, ) @functools.lru_cache(maxsize=8) def _shared_recorder(root: Path) -> BundleRecorder: prepared = prepare_bundle(root) if isinstance(prepared, UnsafeBundleDir): raise ValueError(f"E2E_FIXTURE_DIR {prepared.path} {prepared.reason}") return prepared @functools.lru_cache(maxsize=8) def _shared_replay_source(root: Path) -> ReplaySource: loaded = load_bundle(root) if isinstance(loaded, UnreadableBundle): raise ValueError(f"cannot replay from {root}: {loaded.reason}") return ReplaySource(bundle=loaded) @functools.lru_cache(maxsize=8) def _shared_edge( mode: Literal["record", "replay"], bundle_dir: Path, bind_host: str, advertise_host: str, forward_timeout: float, ) -> ProviderEdge: backend: Final[EdgeBackend] = ( RecordEdge(recorder=_shared_recorder(bundle_dir), lock=threading.Lock()) if mode == "record" else ReplayEdge(source=_shared_replay_source(bundle_dir)) ) return start_provider_edge( backend, mounts=EDGE_MOUNTS, bind_host=bind_host, advertise_host=advertise_host, forward_timeout=forward_timeout, ).edge def replay_leftover_error(*, mode_raw: str, bundle_dir: Path, test_key: str) -> str | None: """Teardown-time completeness check: in replay mode a passed test with unconsumed recorded interactions must fail instead of passing against a recording it no longer matches. Inert in every other mode.""" if parse_fixture_mode(mode_raw) != "replay": return None return _shared_replay_source(bundle_dir).leftover_error(test_key) def provider_edge_api_base( mount: str, *, mode_raw: str, bundle_dir: Path, bind_host: str, advertise_host: str, forward_timeout: float = 60.0, ) -> str | None: """The api_base a suite gives an edge-wired deployment: None in live mode (the deployment keeps its real provider api_base) and the process-wide edge server's mount URL in record and replay, booting the server on first use.""" mode: Final = parse_fixture_mode(mode_raw) match mode: case InvalidFixtureMode(value=value): raise ValueError(f"E2E_FIXTURE_MODE={value!r} is not one of {', '.join(FIXTURE_MODES)}") case "live": return None case "record" | "replay": if mount not in EDGE_MOUNTS: raise ValueError( f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}" ) return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout).api_base(mount) case _: assert_never(mode)