diff --git a/tests/_fake_openai_endpoint_server.py b/tests/_fake_openai_endpoint_server.py index 409f569070b..caf3fb5ba2a 100644 --- a/tests/_fake_openai_endpoint_server.py +++ b/tests/_fake_openai_endpoint_server.py @@ -17,6 +17,7 @@ something to trip on. from __future__ import annotations +import asyncio import json import time import uuid @@ -30,6 +31,8 @@ from starlette.routing import Route _CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint." _RATE_LIMIT_MODEL: Final = "429" +_SLOW_MODEL: Final = "slow-endpoint" +_SLOW_RESPONSE_SECONDS: Final = 3.0 _PROMPT_TOKENS: Final = 20 _COMPLETION_TOKENS: Final = 20 @@ -123,6 +126,8 @@ async def chat_completions(request: Request) -> Response: model = _requested_model(body) if model == _RATE_LIMIT_MODEL: return _rate_limit_response(model) + if model == _SLOW_MODEL: + await asyncio.sleep(_SLOW_RESPONSE_SECONDS) if _wants_stream(body): return StreamingResponse( _chat_completion_stream(model, _wants_stream_usage(body)), @@ -175,6 +180,8 @@ async def completions(request: Request) -> Response: model = _requested_model(body) if model == _RATE_LIMIT_MODEL: return _rate_limit_response(model) + if model == _SLOW_MODEL: + await asyncio.sleep(_SLOW_RESPONSE_SECONDS) if _wants_stream(body): return StreamingResponse( _text_completion_stream(model, _wants_stream_usage(body)), @@ -197,6 +204,22 @@ async def embeddings(request: Request) -> Response: ) +async def triton_embeddings(_request: Request) -> Response: + return JSONResponse( + { + "model_name": "my-triton-model", + "outputs": [ + { + "name": "output", + "datatype": "FP32", + "shape": [1, 2], + "data": [0.1, 0.2], + } + ], + } + ) + + async def list_models(_request: Request) -> Response: return JSONResponse( { @@ -223,6 +246,7 @@ app = Starlette( Route("/v1/completions", completions, methods=["POST"]), Route("/embeddings", embeddings, methods=["POST"]), Route("/v1/embeddings", embeddings, methods=["POST"]), + Route("/triton/embeddings", triton_embeddings, methods=["POST"]), Route("/models", list_models, methods=["GET"]), Route("/v1/models", list_models, methods=["GET"]), ] diff --git a/tests/fake_openai_endpoint.py b/tests/fake_openai_endpoint.py new file mode 100644 index 00000000000..2e2439eb57f --- /dev/null +++ b/tests/fake_openai_endpoint.py @@ -0,0 +1,95 @@ +"""Shared access to the canned OpenAI mock used across the test suite. + +Tests that need a fake OpenAI-shaped endpoint point ``api_base`` at +``FAKE_OPENAI_API_BASE`` instead of a hosted URL, so the suite never depends on +an external service staying up. ``ensure_fake_openai_endpoint`` returns a base +URL that is actually serving: it reuses a server already listening on that base +(it answers ``/health``) and otherwise spawns ``_fake_openai_endpoint_server.py`` +detached from the spawning process so it survives until the host goes away. + +We deliberately do not register an interpreter-exit teardown. Under +``pytest-xdist`` the spawn happens inside whichever worker wins the race, and +workers exit independently as their queues drain; a per-worker ``atexit`` would +terminate the shared mock mid-session for the workers still running. In CI the +container is ephemeral, and locally the next run reuses the still-healthy server +via ``/health`` (or a developer can free the port manually). + +The resolved base honors a ``FAKE_OPENAI_API_BASE`` env var only when it points +at a loopback host; a remote value (CI sets it to the old hosted mock) is ignored +in favor of the local default. Otherwise this helper would try to bind a local +server to a remote host and time out, which is the exact external dependency the +local mock exists to remove. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from functools import lru_cache +from pathlib import Path +from typing import Final +from urllib.error import URLError +from urllib.parse import urlsplit +from urllib.request import urlopen + +_LOCAL_DEFAULT: Final = "http://127.0.0.1:8190" +_LOOPBACK_HOSTS: Final = frozenset({"127.0.0.1", "localhost", "::1"}) + + +def _resolve_base() -> str: + configured = os.environ.get("FAKE_OPENAI_API_BASE") + if configured and urlsplit(configured).hostname in _LOOPBACK_HOSTS: + return configured + return _LOCAL_DEFAULT + + +FAKE_OPENAI_API_BASE: Final = _resolve_base() + +_SERVER_SCRIPT: Final = ( + Path(__file__).resolve().parent / "_fake_openai_endpoint_server.py" +) +_HEALTH_TIMEOUT_SECONDS: Final = 30.0 + + +def _is_healthy(base: str) -> bool: + try: + with urlopen(f"{base.rstrip('/')}/health", timeout=1) as response: + return response.status == 200 + except (URLError, OSError): + return False + + +def _wait_until_healthy(base: str) -> None: + deadline = time.monotonic() + _HEALTH_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if _is_healthy(base): + return + time.sleep(0.5) + raise RuntimeError( + f"fake OpenAI endpoint at {base} did not become healthy within {_HEALTH_TIMEOUT_SECONDS}s" + ) + + +@lru_cache(maxsize=1) +def ensure_fake_openai_endpoint() -> str: + base = FAKE_OPENAI_API_BASE + if _is_healthy(base): + return base + parts = urlsplit(base) + subprocess.Popen( + [ + sys.executable, + str(_SERVER_SCRIPT), + "--host", + parts.hostname or "127.0.0.1", + "--port", + str(parts.port or 8190), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + _wait_until_healthy(base) + return base diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index dba3812ee1c..77fcb46a2b1 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -31,6 +31,14 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 reset_vcr_diag_dir, vcr_config_dict, ) +from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 + + +@pytest.fixture(scope="session", autouse=True) +def fake_openai_endpoint(): + ensure_fake_openai_endpoint() + yield + # Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles # the vast majority of respx-vs-vcrpy conflicts automatically. The only diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index 2d1ca39e1dc..21887e8d848 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -19,6 +19,8 @@ import pytest from litellm.llms.triton.embedding.transformation import TritonEmbeddingConfig import litellm +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + def test_split_embedding_by_shape_passes(): try: @@ -360,7 +362,7 @@ async def test_triton_embeddings(): litellm.set_verbose = True response = await litellm.aembedding( model="triton/my-triton-model", - api_base="https://exampleopenaiendpoint-production.up.railway.app/triton/embeddings", + api_base=f"{FAKE_OPENAI_API_BASE}/triton/embeddings", input=["good morning from litellm"], ) print(f"response: {response}") diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index d45caec22d8..2bda6f0d22b 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -47,6 +47,14 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 reset_vcr_diag_dir, vcr_config_dict, ) +from tests.fake_openai_endpoint import ensure_fake_openai_endpoint # noqa: E402 + + +@pytest.fixture(scope="session", autouse=True) +def fake_openai_endpoint(): + ensure_fake_openai_endpoint() + yield + # Per-item respx detection (``apply_vcr_auto_marker_to_items``) auto-skips # tests whose ``@pytest.mark.respx`` marker or ``respx_mock`` fixture @@ -62,6 +70,8 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 _VCR_INCOMPATIBLE_FILES = frozenset( { "test_router_caching.py", + # Hits the local fake OpenAI endpoint on 127.0.0.1; nothing to record. + "test_fake_openai_endpoint.py", } ) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index cce6d33e799..616d8b94e6a 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -24,6 +24,8 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + # litellm.num_retries=3 litellm.cache = None @@ -1343,7 +1345,7 @@ def test_lm_studio_completion(monkeypatch): messages=[ {"role": "user", "content": "What's the weather like in San Francisco?"} ], - api_base="https://exampleopenaiendpoint-production.up.railway.app/", + api_base=FAKE_OPENAI_API_BASE, ) except litellm.AuthenticationError as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/local_testing/test_fake_openai_endpoint.py b/tests/local_testing/test_fake_openai_endpoint.py new file mode 100644 index 00000000000..79d8b4f97e3 --- /dev/null +++ b/tests/local_testing/test_fake_openai_endpoint.py @@ -0,0 +1,102 @@ +"""Regression tests for the canned OpenAI mock that decouples the suite from a +hosted endpoint (see ``tests/fake_openai_endpoint.py`` and +``tests/_fake_openai_endpoint_server.py``). + +Before this, router/completion/triton tests hardcoded a shared Railway mock as +their ``api_base``; when that single deployment went down, unrelated CI jobs +failed with ``404 Application not found``. These tests pin the two behaviors +those tests now rely on from the local stand-in (the Triton embeddings shape and +the ``slow-endpoint`` delay) and guard against the dead host creeping back in. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import httpx +import pytest + +from tests.fake_openai_endpoint import ( + _LOCAL_DEFAULT, + _resolve_base, + ensure_fake_openai_endpoint, +) + +_REPO_ROOT = Path(__file__).resolve().parents[2] + +_MIGRATED_FILES = ( + "tests/llm_translation/test_triton.py", + "tests/local_testing/test_router.py", + "tests/local_testing/test_router_custom_routing.py", + "tests/local_testing/test_router_fallback_handlers.py", + "tests/local_testing/test_router_fallbacks.py", + "tests/local_testing/test_secret_detect_hook.py", + "tests/local_testing/test_lowest_latency_routing.py", + "tests/local_testing/test_completion.py", +) + +# A live ``railway.app/`` or ``railway.app"`` api_base reappearing in a migrated +# file would re-couple CI to an external service's uptime. The deliberately +# broken ``...railway.appzzzzz`` fallback URL is not a live host, so it does not +# match. +_LIVE_HOSTED_MOCK = re.compile(r"railway\.app(?:/|\")") + + +def test_chat_completion_shape(): + base = ensure_fake_openai_endpoint() + response = httpx.post( + f"{base}/v1/chat/completions", + json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}, + timeout=10, + ) + assert response.status_code == 200 + body = response.json() + assert body["choices"][0]["message"]["content"] + assert body["usage"]["total_tokens"] == 40 + + +def test_triton_embeddings_route(): + base = ensure_fake_openai_endpoint() + response = httpx.post(f"{base}/triton/embeddings", json={"inputs": []}, timeout=10) + assert response.status_code == 200 + output = response.json()["outputs"][0] + assert output["shape"] == [1, 2] + assert output["data"] == [0.1, 0.2] + + +def test_slow_model_blocks_past_client_timeout(): + base = ensure_fake_openai_endpoint() + with pytest.raises(httpx.TimeoutException): + httpx.post( + f"{base}/v1/chat/completions", + json={ + "model": "slow-endpoint", + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=0.5, + ) + + +def test_remote_env_base_resolves_to_local(monkeypatch): + monkeypatch.setenv( + "FAKE_OPENAI_API_BASE", + "https://exampleopenaiendpoint-production.up.railway.app", + ) + assert _resolve_base() == _LOCAL_DEFAULT + + +@pytest.mark.parametrize("host", ["127.0.0.1", "localhost", "[::1]"]) +def test_loopback_env_base_is_honored(monkeypatch, host): + base = f"http://{host}:9191" + monkeypatch.setenv("FAKE_OPENAI_API_BASE", base) + assert _resolve_base() == base + + +@pytest.mark.parametrize("relative_path", _MIGRATED_FILES) +def test_migrated_files_have_no_live_hosted_mock(relative_path): + source = (_REPO_ROOT / relative_path).read_text() + assert not _LIVE_HOSTED_MOCK.search(source), ( + f"{relative_path} references the hosted Railway mock; point api_base at " + "FAKE_OPENAI_API_BASE so CI does not depend on an external service" + ) diff --git a/tests/local_testing/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py index 90913499e55..ac84b3ec5e9 100644 --- a/tests/local_testing/test_lowest_latency_routing.py +++ b/tests/local_testing/test_lowest_latency_routing.py @@ -25,6 +25,8 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + ### UNIT TESTS FOR LATENCY ROUTING ### @@ -591,7 +593,7 @@ async def test_lowest_latency_routing_with_timeouts(): "model_name": "azure-model", "litellm_params": { "model": "openai/slow-endpoint", - "api_base": "https://exampleopenaiendpoint-production-c715.up.railway.app/", # If you are Krrish, this is OpenAI Endpoint3 on our Railway endpoint :) + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "slow-endpoint"}, @@ -600,7 +602,7 @@ async def test_lowest_latency_routing_with_timeouts(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint"}, @@ -666,7 +668,7 @@ async def test_lowest_latency_routing_first_pick(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint"}, @@ -675,7 +677,7 @@ async def test_lowest_latency_routing_first_pick(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint-2", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint-2"}, @@ -684,7 +686,7 @@ async def test_lowest_latency_routing_first_pick(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint-2", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint-3"}, @@ -693,7 +695,7 @@ async def test_lowest_latency_routing_first_pick(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint-2", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint-4"}, diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index 6f1e367760f..2ee1aee9710 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -34,6 +34,8 @@ from litellm.router_utils.cooldown_handlers import ( ) from litellm.types.router import DeploymentTypedDict +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + load_dotenv() @@ -144,14 +146,14 @@ async def test_router_provider_wildcard_routing_regex(): "model_name": "openai/fo::*:static::*", "litellm_params": { "model": "openai/fo::*:static::*", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, }, }, { "model_name": "openai/foo3::hello::*", "litellm_params": { "model": "openai/foo3::hello::*", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, }, }, ] @@ -1639,10 +1641,7 @@ async def test_router_text_completion_client(): "litellm_params": { "model": "text-completion-openai/gpt-3.5-turbo-instruct", "api_key": os.getenv("OPENAI_API_KEY", None), - "api_base": os.getenv( - "FAKE_OPENAI_API_BASE", - "https://exampleopenaiendpoint-production.up.railway.app/", - ), + "api_base": FAKE_OPENAI_API_BASE, }, } ] diff --git a/tests/local_testing/test_router_custom_routing.py b/tests/local_testing/test_router_custom_routing.py index 3f829a13c02..698ccac10dc 100644 --- a/tests/local_testing/test_router_custom_routing.py +++ b/tests/local_testing/test_router_custom_routing.py @@ -18,6 +18,8 @@ import litellm from litellm import Router from litellm.router import CustomRoutingStrategyBase +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + def _create_router(): return Router( @@ -26,7 +28,7 @@ def _create_router(): "model_name": "azure-model", "litellm_params": { "model": "openai/very-special-endpoint", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "very-special-endpoint"}, @@ -35,7 +37,7 @@ def _create_router(): "model_name": "azure-model", "litellm_params": { "model": "openai/fast-endpoint", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "fake-key", }, "model_info": {"id": "fast-endpoint"}, diff --git a/tests/local_testing/test_router_fallback_handlers.py b/tests/local_testing/test_router_fallback_handlers.py index bc9b42f5a05..0bd455463b7 100644 --- a/tests/local_testing/test_router_fallback_handlers.py +++ b/tests/local_testing/test_router_fallback_handlers.py @@ -22,6 +22,8 @@ from litellm.router_utils.fallback_event_handlers import ( log_failure_fallback_event, ) +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + # Helper function to create a Router instance def create_test_router(): @@ -68,7 +70,7 @@ def create_test_router_2(): "litellm_params": { "model": "openai/fake-openai-endpoint-2", "api_key": "working-key-since-this-is-fake-endpoint", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, }, }, ], @@ -308,7 +310,4 @@ async def test_multiple_fallbacks(function_name): print(result._hidden_params) - assert ( - result._hidden_params["api_base"] - == "https://exampleopenaiendpoint-production.up.railway.app/" - ) + assert result._hidden_params["api_base"] == FAKE_OPENAI_API_BASE diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index a14e53adbc4..6547a3eb663 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -18,6 +18,8 @@ import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + class MyCustomHandler(CustomLogger): success: bool = False @@ -1345,7 +1347,7 @@ def test_router_fallbacks_with_custom_model_costs(): "model": "openai/claude-sonnet-4-5-20250929", "input_cost_per_token": 0.000003, # 3$/M "output_cost_per_token": 0.000015, # 15$/M - "api_base": "https://exampleopenaiendpoint-production.up.railway.app", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "my-fake-key", "mock_response": "Hello! How can I help you today?", }, @@ -1594,7 +1596,7 @@ async def test_router_attempted_fallbacks_in_response(expected_attempted_fallbac "litellm_params": { "model": "openai/working-fake-endpoint", "api_key": "my-fake-key", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app", + "api_base": FAKE_OPENAI_API_BASE, }, }, { diff --git a/tests/local_testing/test_secret_detect_hook.py b/tests/local_testing/test_secret_detect_hook.py index 8340fb91482..57b55bd2689 100644 --- a/tests/local_testing/test_secret_detect_hook.py +++ b/tests/local_testing/test_secret_detect_hook.py @@ -36,6 +36,8 @@ from litellm.proxy.proxy_server import chat_completion from litellm.proxy.utils import ProxyLogging, hash_token from litellm.router import Router +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE + ### UNIT TESTS FOR OpenAI Moderation ### @@ -246,7 +248,7 @@ router = Router( "model_name": "fake-model", "litellm_params": { "model": "openai/fake", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "api_base": FAKE_OPENAI_API_BASE, "api_key": "sk-12345", }, }