test: fix shared-provider discovery, Codex catalog size and generated master key mismatches in CircleCI suites (#44905)

* test(integration): isolate Codex catalog and provider discovery fixtures

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): keep model discovery constant import-safe

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: align test keys and CI env with generated master keys

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: mateo <mateo@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -3182,6 +3182,11 @@ jobs:
       E2E_FIXTURE_MODE: live
     steps:
       - checkout
+      - run:
+          name: Generate LiteLLM master key
+          command: |
+            key="$(openssl rand -hex 16)"
+            printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
       - skip_if_unrelated_changes:
           category: provider-harness
       - setup_litellm_test_deps

tests/integration/_support/openai_wire.py

@@ -7,8 +7,9 @@ from typing import Final
 from integration._support.wire import Reply, Request, Wire
 from pydantic import JsonValue
 
+from tests.integration._support.provider import MODEL_DISCOVERY
+
 _USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}
-MODEL_DISCOVERY: Final = ("GET", "/v1/models")
 
 
 def answering_model_discovery(respond: Callable[[Request], Reply]) -> Callable[[Request], Reply]:

tests/integration/_support/provider.py

@@ -3,6 +3,7 @@
 Deployments in `proxy_config.yaml` point at `PROVIDER_URL`, so one server answers for all of them. A test
 queues the replies it expects with `expect` and reads what the proxy sent with `received`. Tests run one at a
 time against it; the `provider` fixture checks nothing is left over between tests.
+The proxy's model-info refresh is answered here.
 """
 
 from __future__ import annotations
@@ -17,6 +18,7 @@ from tests.integration._support.wire import Reply, Request, Wire, wire_server
 
 PROVIDER_PORT: Final = 8191
 PROVIDER_URL: Final = f"http://127.0.0.1:{PROVIDER_PORT}"
+MODEL_DISCOVERY: Final = ("GET", "/v1/models")
 _UNQUEUED: Final = Reply(status=500, body=b'{"error": "the shared fake provider has no reply queued for this request"}')
 
 
@@ -30,7 +32,9 @@ class SharedProvider:
         self.replies.extend(replies)
 
     def received(self) -> tuple[Request, ...]:
-        return self.wire.drain()
+        return tuple(
+            request for request in self.wire.drain() if (request.method, request.target) != MODEL_DISCOVERY
+        )
 
 
 @contextmanager
@@ -38,6 +42,8 @@ def shared_provider() -> Iterator[SharedProvider]:
     replies: Final[deque[Reply]] = deque()
 
     def respond(request: Request) -> Reply:
+        if (request.method, request.target) == MODEL_DISCOVERY:
+            return Reply(body=b'{"object":"list","data":[]}')
         return replies.popleft() if replies else _UNQUEUED
 
     with wire_server(respond, port=PROVIDER_PORT) as wire:

tests/integration/compatibility/test_codex_model_catalog.py

@@ -3,21 +3,33 @@ import json
 import os
 import time
 import uuid
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from concurrent.futures import ThreadPoolExecutor
 from hashlib import sha256
 from itertools import chain
 from pathlib import Path
+from types import MappingProxyType
 from typing import Final, TypeVar
 
 import anthropic
 import httpx
 import openai
+import pytest
+import yaml
 from pydantic import JsonValue
 
 from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS
-from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
+from tests.integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    Scenario,
+    eventually,
+    gateway_from_environment,
+    object_value,
+    string_value,
+)
 from tests.integration._support.database import read_rows
+from tests.integration._support.process import owned_proxy
 from tests.integration._support.wire import Reply, Request, Wire, wire_server
 
 T = TypeVar("T")
@@ -36,6 +48,25 @@ ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"}
 USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}
 
 
+def _config_without_model_list(directory: Path) -> Path:
+    shared_config: Final = object_value(yaml.safe_load((ROOT / "tests/integration/proxy_config.yaml").read_text()))
+    config_without_models: Final[Mapping[str, JsonValue]] = MappingProxyType(
+        {name: value for name, value in shared_config.items() if name != "model_list"}
+    )
+    path: Final = directory / "proxy_config_without_model_list.yaml"
+    path.write_text(yaml.safe_dump({**config_without_models}))
+    return path
+
+
+@pytest.fixture(scope="module")
+def gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
+    with gateway_from_environment() as parent_gateway:
+        directory: Final = tmp_path_factory.mktemp("codex_model_catalog")
+        config: Final = _config_without_model_list(directory)
+        with owned_proxy(parent_gateway, directory, {}, config=config, workers=PROXY_WORKERS) as isolated_gateway:
+            yield isolated_gateway
+
+
 def _generic_tier(identity: str) -> dict[str, JsonValue]:
     return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}
 

tests/litellm_utils_tests/test_secret_manager.py

@@ -1,4 +1,5 @@
 import base64
+import hashlib
 import os
 import time
 import traceback
@@ -10,6 +11,7 @@ import json
 load_dotenv()
 import tempfile
 from uuid import uuid4
+from typing import Final
 
 import pytest
 import litellm
@@ -23,6 +25,8 @@ from litellm.secret_managers.main import (
 )
 from unittest.mock import AsyncMock, patch, MagicMock
 
+_AWS_FIXTURE_MASTER_KEY_SHA256: Final = "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"
+
 
 def load_vertex_ai_credentials():
     # Define the path to the vertex_key.json file
@@ -76,7 +80,9 @@ def test_aws_secret_manager():
     # cast json to dict
     secret_val = json.loads(secret_val)
 
-    assert secret_val["litellm_master_key"] == os.environ["LITELLM_MASTER_KEY"]
+    assert (
+        hashlib.sha256(secret_val["litellm_master_key"].encode()).hexdigest() == _AWS_FIXTURE_MASTER_KEY_SHA256
+    ), "Expected the fixture value stored in the CI AWS account's litellm_master_key secret"
 
 
 def redact_oidc_signature(secret_val):

tests/mcp_tests/test_proxy_mcp_e2e.py

@@ -390,7 +390,7 @@ async def _http_streams(url: str, headers: dict[str, str]):
 @pytest.mark.asyncio
 async def test_unchanged_sdk1_langchain_peer_can_list_and_call(proxy_server_url: str) -> None:
     script = """
-import asyncio, json, sys
+import asyncio, json, os, sys
 from mcp import ClientSession
 from mcp.client.streamable_http import streamablehttp_client
 from langchain_mcp_adapters.tools import load_mcp_tools
@@ -706,7 +706,7 @@ class TestProxyMcpSchemaDiscoveryMode:
 async def authorize_proxy_key(request: Request, api_key: str) -> UserAPIKeyAuth:
     permissions = {
         "sk-schema": LiteLLM_ObjectPermissionTable(object_permission_id="schema", mcp_servers=["schema"]),
-        "sk-9876": LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]),
+        MASTER_KEY: LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]),
         "sk-restricted": LiteLLM_ObjectPermissionTable(
             object_permission_id="restricted", mcp_servers=["math_restricted"]
         ),
@@ -746,7 +746,7 @@ proxy_call_recorder = ProxyCallRecorder()
 
 
 @asynccontextmanager
-async def _scoped_session(url: str, key: str = "sk-9876", **headers: str) -> typing.AsyncIterator[ClientSession]:
+async def _scoped_session(url: str, key: str = MASTER_KEY, **headers: str) -> typing.AsyncIterator[ClientSession]:
     async with asyncio.timeout(30):
         async with _proxy_session(url, Authorization=f"Bearer {key}", **headers) as (read, write):
             async with ClientSession(read, write) as session:

tests/unified_google_tests/base_google_genai_proxy_sdk_test.py

@@ -6,6 +6,8 @@ from typing import Any, Dict, List, Optional
 
 import pytest
 
+from tests._master_key import MASTER_KEY
+
 try:
     from google import genai
     from google.genai import types
@@ -14,7 +16,6 @@ try:
 except ImportError:
     GOOGLE_GENAI_SDK_AVAILABLE = False
 
-MASTER_KEY = "sk-unified-google-tests-4f9b2c7d8e1a"
 PROMPT = "Reply with only the single word: pong"