From 758f72e41f66a22e9d05f930b38efc8c8e7e2294 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 1 May 2026 10:12:50 -0700 Subject: [PATCH] tests(e2e-cassette-proxy): add mitm.it to passthrough hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mitmproxy serves its current-instance CA certificate at the magic host ``mitm.it/cert/pem``. The CI pipeline downloads that CA in the ``Fetch CA from cassette-proxy`` step and trusts it (certifi append + system trust store) so subsequent in-process tests can validate mitmproxy's MITM leaf certs. Once the previous commit fixed the replay path so cache hits actually serve, the very first ``mitm.it/cert/pem`` lookup hit a stale entry from a prior CI run — the proxy handed back a CA generated by a *previous* mitmdump instance with a different private key. The CI then trusted that stale CA, mitmproxy in the *current* run signed leaf certs with its fresh CA, and every TLS handshake failed: SSL: CERTIFICATE_VERIFY_FAILED: certificate verify failed: authority and subject key identifier mismatch This took down langfuse_logging_unit_tests (at ``test_embedding.py`` collection), search_testing, and image_gen_testing. Adding ``mitm.it`` to the passthrough host list short-circuits in ``_should_skip`` before any Redis lookup, so the CA is always served fresh from the running mitmdump instance. The poisoned key in the shared dev Redis was deleted manually before pushing this commit. Adds a regression test that exercises both hooks: ``request()`` must return without setting ``flow.response``, and ``response()`` must not persist anything to Redis. --- tests/e2e_cassette_proxy/addon.py | 17 ++++++-- .../e2e_cassette_proxy/test_addon.py | 41 +++++++++++++++++-- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/e2e_cassette_proxy/addon.py b/tests/e2e_cassette_proxy/addon.py index ed23c6bcc80..84cd9d334d1 100644 --- a/tests/e2e_cassette_proxy/addon.py +++ b/tests/e2e_cassette_proxy/addon.py @@ -42,14 +42,25 @@ if not _log.handlers: _log.propagate = False -# Hosts we should *never* cache — Redis itself, the proxy admin UI, -# anything pointed at localhost. Extended via env var -# ``LITELLM_E2E_CASS_PASSTHROUGH_HOSTS`` (comma-separated). +# Hosts we should *never* cache. Three categories: +# +# - Loopback / sidecar: localhost, 127.0.0.1, host.docker.internal — +# the test harness itself, postgres, redis, etc. +# - The proxy's own admin UI (``mitm.it``): mitmproxy serves its CA +# certificate from this magic host. The CA is regenerated per +# ``mitmdump`` instance, so caching its response would hand a stale +# CA to the next CI run, which would then trust leaf certs signed +# by a *different* CA → ``SSL: CERTIFICATE_VERIFY_FAILED: +# authority and subject key identifier mismatch``. +# +# Extend at runtime via ``LITELLM_E2E_CASS_PASSTHROUGH_HOSTS`` +# (comma-separated). _PASSTHROUGH_HOSTS = { "localhost", "127.0.0.1", "0.0.0.0", "host.docker.internal", + "mitm.it", } diff --git a/tests/test_litellm/e2e_cassette_proxy/test_addon.py b/tests/test_litellm/e2e_cassette_proxy/test_addon.py index 96d43e270af..c1fd5640f01 100644 --- a/tests/test_litellm/e2e_cassette_proxy/test_addon.py +++ b/tests/test_litellm/e2e_cassette_proxy/test_addon.py @@ -137,10 +137,7 @@ sys.modules.setdefault("mitmproxy", _mitmproxy_pkg) sys.modules.setdefault("mitmproxy.http", _http_mod) from tests.e2e_cassette_proxy.addon import CassetteAddon # noqa: E402 -from tests.e2e_cassette_proxy.redis_store import ( # noqa: E402 - CachedResponse, - RedisCassetteStore, -) +from tests.e2e_cassette_proxy.redis_store import RedisCassetteStore # noqa: E402 def _addon_with_fake_redis(): @@ -233,6 +230,42 @@ def test_should_skip_passthrough_hosts_completely(): assert addon._stats["skipped"] == 1 +def test_should_skip_mitm_it_so_ca_cert_is_not_cached(): + """``mitm.it/cert/pem`` serves the proxy's *current-instance* CA. If we + cached it, the next mitmdump run would serve a stale CA from Redis, + and clients trusting that stale CA would reject leaf certs signed by + the new run's CA: ``SSL: CERTIFICATE_VERIFY_FAILED: authority and + subject key identifier mismatch``. So ``mitm.it`` must be in the + passthrough list end-to-end (request *and* response paths).""" + fake, addon = _addon_with_fake_redis() + + cert_req = _FakeRequest( + method="GET", + url="http://mitm.it/cert/pem", + body=b"", + headers={"accept": "*/*"}, + host="mitm.it", + path="/cert/pem", + ) + flow = _FakeFlow(cert_req) + addon.request(flow) + # Request hook short-circuits before any Redis lookup. + assert flow.response is None + assert addon._stats["skipped"] == 1 + assert addon._stats["miss"] == 0 + + # Even if the response hook fires (mitmproxy fetched the real CA from + # the running instance), it must not be persisted to Redis. + flow.response = _FakeResponse( + 200, + body=b"-----BEGIN CERTIFICATE-----\nMIIabc...\n-----END CERTIFICATE-----\n", + headers={"content-type": "application/x-x509-ca-cert"}, + ) + addon.response(flow) + assert addon._stats["stored"] == 0 + assert len(fake.keys("*")) == 0 + + def test_replay_only_should_serve_599_on_miss(): fake, _ = _addon_with_fake_redis() addon = CassetteAddon(store=RedisCassetteStore(client=fake))