diff --git a/litellm/proxy/common_utils/codex_catalog_source.py b/litellm/proxy/common_utils/codex_catalog_source.py new file mode 100644 index 00000000000..994ffe8e72e --- /dev/null +++ b/litellm/proxy/common_utils/codex_catalog_source.py @@ -0,0 +1,181 @@ +"""Codex's own catalog for the Codex release asking for `/v1/models`, fetched from that release's tag on GitHub. + +Codex fetches its catalog from chatgpt.com, which needs a ChatGPT login the proxy never sees, so the proxy reads +the same two files at the release's public tag instead, and falls back to the copy bundled with LiteLLM when the +fetch fails, the release has no tag (dev builds report 0.0.0) or ``LITELLM_OFFLINE_MODE`` is on. +""" + +from __future__ import annotations + +import asyncio +import re +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from functools import cache +from types import MappingProxyType +from typing import Final, Protocol, TypeVar + +import httpx + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared factory has untyped optional parameters +) +from litellm.proxy.common_utils.codex_model_catalog import ( + CODEX_BUNDLED_MODELS_PATH, + CodexStockCatalog, + CodexStockModel, + bundled_codex_models, + codex_base_instructions, +) +from litellm.types.llms.custom_http import httpxSpecialProvider + +CODEX_RELEASE_FILE_URL: Final = ( + "https://raw.githubusercontent.com/openai/codex/rust-v{version}/codex-rs/models-manager/{file}" +) +CODEX_FETCH_TIMEOUT_SECONDS: Final = 2.0 +CODEX_FETCH_FAILURE_TTL_SECONDS: Final = 3600.0 +CODEX_CACHED_VERSIONS: Final = 8 +CODEX_REMEMBERED_FAILURES: Final = 256 +CODEX_FETCHES_PER_MINUTE: Final = 6 + +_RELEASE_VERSION: Final = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.]+)?") + + +@dataclass(frozen=True, slots=True) +class CodexStock: + models: Mapping[str, CodexStockModel] + instructions: str + + +_Value: Final = TypeVar("_Value") + + +class CodexReleaseClient(Protocol): + def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ... + + +@cache +def bundled_codex_stock() -> CodexStock: + return CodexStock(models=bundled_codex_models(), instructions=codex_base_instructions()) + + +async def _release_file(client: CodexReleaseClient, version: str, file: str, timeout: float) -> str: + response: Final = await client.get(CODEX_RELEASE_FILE_URL.format(version=version, file=file), timeout=timeout) + return response.raise_for_status().content.decode("utf-8") + + +async def _release_stock(client: CodexReleaseClient, version: str, timeout: float) -> CodexStock: + models_json, instructions = await asyncio.wait_for( + asyncio.gather( + _release_file(client, version, "models.json", timeout), + _release_file(client, version, "prompt.md", timeout), + ), + timeout, + ) + catalog: Final = CodexStockCatalog.model_validate_json(models_json) + return CodexStock( + models=MappingProxyType({model.slug: model for model in catalog.models}), instructions=instructions + ) + + +class CodexCatalogSource: + """Codex's catalog per release: the one fetched from GitHub, else the bundled one. + + A fetched catalog is kept for good, a failed fetch is remembered for `failure_ttl` seconds so a blocked egress + costs one timeout per worker per version per hour. Only the `max_versions` most recently fetched catalogs and + the `max_failures` most recent failures are kept, apart, so versions that fail never push out ones that were + fetched. + Requests for a version already being fetched wait on that fetch, and a worker starts at most + `fetches_per_minute` fetches, serving the bundled catalog past that, so a caller cycling made-up versions + cannot turn every request into GitHub requests.""" + + def __init__( + self, + client: Callable[[], CodexReleaseClient], + *, + offline: Callable[[], bool] = is_offline_mode, + clock: Callable[[], float] = time.monotonic, + timeout: float = CODEX_FETCH_TIMEOUT_SECONDS, + failure_ttl: float = CODEX_FETCH_FAILURE_TTL_SECONDS, + max_versions: int = CODEX_CACHED_VERSIONS, + max_failures: int = CODEX_REMEMBERED_FAILURES, + fetches_per_minute: int = CODEX_FETCHES_PER_MINUTE, + ) -> None: + self._client: Final = client + self._offline: Final = offline + self._clock: Final = clock + self._timeout: Final = timeout + self._failure_ttl: Final = failure_ttl + self._max_versions: Final = max_versions + self._max_failures: Final = max_failures + self._fetches_per_minute: Final = fetches_per_minute + self._fetched: Mapping[str, CodexStock] = MappingProxyType({}) + self._failed_until: Mapping[str, float] = MappingProxyType({}) + self._in_flight: Mapping[str, asyncio.Future[CodexStock]] = MappingProxyType({}) + self._fetch_starts: tuple[float, ...] = () + + async def stock(self, client_version: str) -> CodexStock: + if self._offline() or _RELEASE_VERSION.fullmatch(client_version) is None: + return bundled_codex_stock() + known: Final = self._fetched.get(client_version) + if known is not None: + return known + if self._clock() < self._failed_until.get(client_version, float("-inf")): + return bundled_codex_stock() + pending: Final = self._in_flight.get(client_version) + if pending is not None: + return await asyncio.shield(pending) + if not self._take_fetch_slot(): + return bundled_codex_stock() + started: Final = asyncio.ensure_future(self._fetch_and_store(client_version)) + self._in_flight = MappingProxyType({**self._in_flight, client_version: started}) + return await asyncio.shield(started) + + def _take_fetch_slot(self) -> bool: + now: Final = self._clock() + recent: Final = tuple(start for start in self._fetch_starts if now - start < 60) + allowed: Final = len(recent) < self._fetches_per_minute + self._fetch_starts = (*recent, now) if allowed else recent + return allowed + + async def _fetch_and_store(self, version: str) -> CodexStock: + try: + fetched: Final = await self._fetch(version) + finally: + self._in_flight = MappingProxyType({key: task for key, task in self._in_flight.items() if key != version}) + if fetched is None: + self._failed_until = _latest( + self._failed_until, version, self._clock() + self._failure_ttl, self._max_failures + ) + return bundled_codex_stock() + self._fetched = _latest(self._fetched, version, fetched, self._max_versions) + return fetched + + async def _fetch(self, version: str) -> CodexStock | None: + try: + return await _release_stock(self._client(), version, self._timeout) + except Exception as e: # noqa: BLE001 # a failed best-effort fetch must never fail the model listing + verbose_proxy_logger.warning( + "Codex %s's model catalog could not be fetched from GitHub (%r), serving the catalog bundled " + "with LiteLLM (%s) for that version for the next %d seconds", + version, + e, + CODEX_BUNDLED_MODELS_PATH.name, + self._failure_ttl, + ) + return None + + +def _latest(stored: Mapping[str, _Value], version: str, value: _Value, limit: int) -> Mapping[str, _Value]: + others: Final = tuple((key, kept) for key, kept in stored.items() if key != version) + return MappingProxyType(dict((*others[max(0, len(others) - limit + 1) :], (version, value)))) + + +def _default_client() -> CodexReleaseClient: + return get_async_httpx_client(llm_provider=httpxSpecialProvider.CodexModelCatalog) + + +codex_catalog_source: Final = CodexCatalogSource(_default_client) diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 01cce9a6ce6..6deee04e273 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -709,10 +709,16 @@ def codex_model_list_body( label_entries: Sequence[tuple[str, str]] | None = None, *, client_version: str | None = None, + stock: Mapping[str, CodexStockModel] | None = None, + instructions: str | None = None, ) -> str: - """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" + """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out. `stock` and + `instructions` are the requesting Codex release's catalog and prompt, the bundled ones when None.""" body: Final = codex_models_response_json( - codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version + codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), + stock=stock, + instructions=instructions, + client_version=client_version, ) if body.left_out: verbose_proxy_logger.warning( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e015740821d..3297a298af4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -522,6 +522,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy +from litellm.proxy.common_utils.codex_catalog_source import codex_catalog_source from litellm.proxy.common_utils.codex_model_catalog import CodexCatalogRow, codex_catalog_rows, codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber @@ -12920,6 +12921,7 @@ async def model_list( admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: + admin_codex_stock: Final = await codex_catalog_source.stock(client_version) return Response( content=await asyncio.to_thread( codex_model_list_body, @@ -12928,6 +12930,8 @@ async def model_list( llm_router, team_id or user_api_key_dict.team_id, client_version=client_version, + stock=admin_codex_stock.models, + instructions=admin_codex_stock.instructions, ), media_type="application/json", ) @@ -12967,6 +12971,7 @@ async def model_list( entries: Final = key_listing.entries labels: Final = key_listing.labels if client_version is not None: + codex_stock: Final = await codex_catalog_source.stock(client_version) return Response( content=await asyncio.to_thread( codex_model_list_body, @@ -12976,6 +12981,8 @@ async def model_list( team_id or user_api_key_dict.team_id, labels, client_version=client_version, + stock=codex_stock.models, + instructions=codex_stock.instructions, ), media_type="application/json", ) diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 47f80c52845..4220a67bddb 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -34,6 +34,7 @@ class httpxSpecialProvider(str, Enum): ROICalculator = "roi_calculator" Sandbox = "sandbox" ModelCostMap = "model_cost_map" + CodexModelCatalog = "codex_model_catalog" PasswordBreachCheck = "password_breach_check" ASGI = "asgi" AgentHarness = "agent_harness" diff --git a/tests/integration/compatibility/test_codex_model_catalog.py b/tests/integration/compatibility/test_codex_model_catalog.py index 6aeeead0d90..a513088d082 100644 --- a/tests/integration/compatibility/test_codex_model_catalog.py +++ b/tests/integration/compatibility/test_codex_model_catalog.py @@ -36,7 +36,7 @@ pytestmark: Final = pytest.mark.shares_worker_state T = TypeVar("T") -CLIENT_VERSION: Final = "0.159.3" +CLIENT_VERSION: Final = "0.166.0" CONSISTENT_READS: Final = 6 CONVERGENCE_SECONDS: Final = 60 RELOAD_MARGIN_SECONDS: Final = 5.0 diff --git a/tests/integration/configuration/test_codex_catalog_boot.py b/tests/integration/configuration/test_codex_catalog_boot.py index 08c60debfcf..6564827e2c6 100644 --- a/tests/integration/configuration/test_codex_catalog_boot.py +++ b/tests/integration/configuration/test_codex_catalog_boot.py @@ -18,7 +18,7 @@ from tests.integration._support.process import owned_proxy, owned_proxy_process pytestmark: Final = pytest.mark.shares_worker_state -CLIENT_VERSION: Final = "0.159.3" +CLIENT_VERSION: Final = "0.166.0" FRESH_CONNECTION: Final = {"Connection": "close"} CONSISTENT_READS: Final = 8 CONVERGENCE_SECONDS: Final = 60 diff --git a/tests/unit/proxy/common_utils/test_codex_catalog_source.py b/tests/unit/proxy/common_utils/test_codex_catalog_source.py new file mode 100644 index 00000000000..e3009e19dd7 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_codex_catalog_source.py @@ -0,0 +1,257 @@ +import asyncio +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Final + +import httpx +import pytest + +from litellm.proxy.common_utils.codex_catalog_source import ( + CODEX_RELEASE_FILE_URL, + CodexCatalogSource, + bundled_codex_stock, +) +from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models + +_TTL: Final = 3600.0 + + +def _release_files(slug: str, prompt: str) -> Mapping[str, bytes]: + entry: Final = {**bundled_codex_models()["gpt-5.5"].model_dump(mode="json", exclude_unset=True), "slug": slug} + return {"models.json": json.dumps({"models": [entry]}).encode(), "prompt.md": prompt.encode()} + + +@dataclass +class _GitHub: + """Serves each tagged release's files, 404 for any other tag, and records every URL asked for.""" + + releases: Mapping[str, Mapping[str, bytes]] + fail: Callable[[httpx.Request], httpx.Response] | None = None + requested: list[str] = field(default_factory=list) + + async def handle(self, request: httpx.Request) -> httpx.Response: + self.requested.append(str(request.url)) + if self.fail is not None: + return self.fail(request) + files: Final = { + CODEX_RELEASE_FILE_URL.format(version=version, file=name): body + for version, release in self.releases.items() + for name, body in release.items() # comprehension-ok: flattens release -> file in one lookup table + } + body: Final = files.get(str(request.url)) + return httpx.Response(404, text="404: Not Found") if body is None else httpx.Response(200, content=body) + + +@dataclass +class _Clock: + now: float = 0.0 + + def __call__(self) -> float: + return self.now + + +def _source( + github: _GitHub, + clock: _Clock | None = None, + *, + failure_ttl: float = _TTL, + max_versions: int = 8, + fetches_per_minute: int = 100, +) -> CodexCatalogSource: + client: Final = httpx.AsyncClient(transport=httpx.MockTransport(github.handle)) + return CodexCatalogSource( + lambda: client, + offline=lambda: False, + clock=clock or _Clock(), + failure_ttl=failure_ttl, + max_versions=max_versions, + fetches_per_minute=fetches_per_minute, + ) + + +async def test_release_catalog_and_prompt_are_served_for_the_asking_codex_version() -> None: + github: Final = _GitHub({"0.170.0": _release_files("gpt-from-0.170.0", "prompt of 0.170.0")}) + + stock: Final = await _source(github).stock("0.170.0") + + assert (list(stock.models), stock.instructions) == (["gpt-from-0.170.0"], "prompt of 0.170.0") + assert stock.models["gpt-from-0.170.0"].display_name == bundled_codex_models()["gpt-5.5"].display_name + + +async def test_each_codex_version_gets_its_own_release_and_a_missing_tag_does_not_spill_over() -> None: + github: Final = _GitHub( + { + "0.166.0": _release_files("gpt-from-0.166.0", "prompt of 0.166.0"), + "0.171.0-alpha.2": _release_files("gpt-from-alpha", "prompt of alpha"), + } + ) + source: Final = _source(github) + + served: Final = {version: await source.stock(version) for version in ("0.166.0", "0.171.0-alpha.2", "0.0.0")} + + assert {version: (list(stock.models), stock.instructions) for version, stock in served.items()} == { + "0.166.0": (["gpt-from-0.166.0"], "prompt of 0.166.0"), + "0.171.0-alpha.2": (["gpt-from-alpha"], "prompt of alpha"), + "0.0.0": (list(bundled_codex_models()), bundled_codex_stock().instructions), + } + + +def _unreachable(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("egress blocked", request=request) + + +def _server_error(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, request=request) + + +@pytest.mark.parametrize("fail", [_unreachable, _server_error], ids=["unreachable", "server-error"]) +async def test_a_failed_fetch_serves_the_bundled_catalog(fail: Callable[[httpx.Request], httpx.Response]) -> None: + github: Final = _GitHub({"0.170.0": _release_files("gpt-from-0.170.0", "p")}, fail=fail) + + assert await _source(github).stock("0.170.0") is bundled_codex_stock() + + +async def test_a_release_that_never_answers_serves_the_bundled_catalog_within_the_timeout() -> None: + async def hang(_: httpx.Request) -> httpx.Response: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + client: Final = httpx.AsyncClient(transport=httpx.MockTransport(hang)) + source: Final = CodexCatalogSource(lambda: client, offline=lambda: False, clock=_Clock(), timeout=0.01) + + assert await asyncio.wait_for(source.stock("0.170.0"), timeout=30) is bundled_codex_stock() + + +@pytest.mark.parametrize( + "files", + [ + {"models.json": b'{"models": [{"slug": "gpt-x"}]}', "prompt.md": b"p"}, + {"models.json": b"rate limited", "prompt.md": b"p"}, + {"models.json": _release_files("gpt-x", "p")["models.json"], "prompt.md": b"\xff\xfe"}, + {"models.json": _release_files("gpt-x", "p")["models.json"]}, + ], + ids=["entry-missing-fields", "not-json", "prompt-not-utf8", "prompt-missing"], +) +async def test_a_release_whose_files_do_not_validate_serves_the_bundled_catalog(files: Mapping[str, bytes]) -> None: + assert await _source(_GitHub({"0.170.0": files})).stock("0.170.0") is bundled_codex_stock() + + +async def test_a_failed_fetch_is_not_retried_until_the_failure_expires() -> None: + github: Final = _GitHub({}, fail=_unreachable) + clock: Final = _Clock() + source: Final = _source(github, clock, failure_ttl=_TTL) + + first: Final = await source.stock("0.170.0") + after_first: Final = len(github.requested) + clock.now = _TTL - 1 + within_ttl: Final = await source.stock("0.170.0") + after_within: Final = len(github.requested) + clock.now = _TTL + 1 + await source.stock("0.170.0") + + assert first is within_ttl is bundled_codex_stock() + assert (after_first > 0, after_within == after_first, len(github.requested) > after_within) == (True, True, True) + + +async def test_a_fetched_release_is_not_fetched_again() -> None: + github: Final = _GitHub({"0.170.0": _release_files("gpt-from-0.170.0", "p")}) + clock: Final = _Clock() + source: Final = _source(github, clock) + + first: Final = await source.stock("0.170.0") + fetches: Final = len(github.requested) + clock.now = _TTL * 100 + + assert await source.stock("0.170.0") is first + assert len(github.requested) == fetches + + +async def test_only_the_most_recent_versions_are_kept() -> None: + github: Final = _GitHub({version: _release_files(version, "p") for version in ("0.1.0", "0.2.0", "0.3.0")}) + source: Final = _source(github, max_versions=2) + for version in ("0.1.0", "0.2.0", "0.3.0"): + await source.stock(version) + fetches: Final = len(github.requested) + + await source.stock("0.3.0") + kept_was_cached: Final = len(github.requested) == fetches + await source.stock("0.1.0") + + assert (kept_was_cached, len(github.requested) > fetches) == (True, True) + + +async def test_versions_that_fail_never_push_out_a_fetched_one() -> None: + github: Final = _GitHub({"0.170.0": _release_files("gpt-from-0.170.0", "p")}) + source: Final = _source(github, max_versions=2) + fetched: Final = await source.stock("0.170.0") + for made_up in ("9.9.1", "9.9.2", "9.9.3"): + await source.stock(made_up) + fetches: Final = len(github.requested) + + assert await source.stock("0.170.0") is fetched + assert len(github.requested) == fetches + + +async def test_a_failed_version_keeps_its_cooldown_past_the_fetched_catalog_cap() -> None: + github: Final = _GitHub({}, fail=_unreachable) + source: Final = _source(github, max_versions=2) + for made_up in ("9.9.1", "9.9.2", "9.9.3", "9.9.4"): + await source.stock(made_up) + fetches: Final = len(github.requested) + + await source.stock("9.9.1") + + assert len(github.requested) == fetches, github.requested + + +async def test_a_worker_starts_at_most_its_fetch_budget_each_minute() -> None: + github: Final = _GitHub({}) + clock: Final = _Clock() + source: Final = _source(github, clock, fetches_per_minute=2) + for made_up in ("9.9.1", "9.9.2", "9.9.3"): + await source.stock(made_up) + within_minute: Final = len(github.requested) + clock.now = 61 + await source.stock("9.9.3") + + assert (within_minute, len(github.requested)) == (4, 6), github.requested + + +async def test_requests_for_a_version_being_fetched_share_that_fetch() -> None: + answer: Final = asyncio.Event() + files: Final = _release_files("gpt-from-0.170.0", "p") + requested: Final[list[str]] = [] + + async def slow_github(request: httpx.Request) -> httpx.Response: + requested.append(str(request.url)) + await answer.wait() + return httpx.Response(200, content=files[str(request.url).rsplit("/", 1)[1]]) + + client: Final = httpx.AsyncClient(transport=httpx.MockTransport(slow_github)) + source: Final = CodexCatalogSource(lambda: client, offline=lambda: False, clock=_Clock()) + waiting: Final = [asyncio.ensure_future(source.stock("0.170.0")) for _ in range(5)] + await asyncio.sleep(0) + answer.set() + served: Final = await asyncio.gather(*waiting) + + assert [list(stock.models) for stock in served] == [["gpt-from-0.170.0"]] * 5 + assert len(requested) == 2, requested + + +async def test_offline_mode_never_fetches(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + github: Final = _GitHub({"0.170.0": _release_files("gpt-from-0.170.0", "p")}) + client: Final = httpx.AsyncClient(transport=httpx.MockTransport(github.handle)) + + stock: Final = await CodexCatalogSource(lambda: client, clock=_Clock()).stock("0.170.0") + + assert (stock, github.requested) == (bundled_codex_stock(), []) + + +@pytest.mark.parametrize("client_version", ["../../main", "0.170.0/../../x", "latest", "", "\u0660.\u0661.\u0662"]) +async def test_a_version_that_is_no_release_tag_is_never_fetched(client_version: str) -> None: + github: Final = _GitHub({}) + + assert await _source(github).stock(client_version) is bundled_codex_stock() + assert github.requested == [] diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index f0b3a8fabc7..d4cc2fcb006 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -1407,6 +1407,34 @@ def test_model_info_that_does_not_validate_declares_no_capability_but_keeps_its_ assert capabilities == CodexDeploymentCapabilities(priced_service_tier_keys=frozenset({priced})) +def test_model_list_body_takes_the_asking_releases_catalog_and_prompt() -> None: + router: Final = litellm.Router( + model_list=[ + {"model_name": "gpt-release-only", "litellm_params": {"model": "openai/gpt-release-only"}}, + {"model_name": "in-house", "litellm_params": {"model": "openai/in-house-coder"}}, + ] + ) + release_entry: Final = bundled_codex_models()["gpt-5.5"].model_copy( + update={"slug": "gpt-release-only", "display_name": "Release Only"} + ) + names: Final = ("gpt-release-only", "in-house") + + body: Final = codex_model_list_body( + _listing(*names), + tuple((name, name) for name in names), + router, + client_version="0.170.0", + stock={"gpt-release-only": release_entry}, + instructions="release prompt", + ) + + served: Final = {entry["slug"]: entry for entry in json.loads(body)["models"]} + assert (served["gpt-release-only"]["display_name"], served["in-house"]["base_instructions"]) == ( + "Release Only", + "release prompt", + ), served + + class _CollidingName(str): def __hash__(self) -> int: return 1