diff --git a/.circleci/config.yml b/.circleci/config.yml index 86d8c8eb127..ee23416ad47 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3182,6 +3182,11 @@ jobs: E2E_FIXTURE_MODE: live steps: - checkout + - run: + name: Generate LiteLLM master key + command: | + key="$(openssl rand -hex 16)" + printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - skip_if_unrelated_changes: category: provider-harness - setup_litellm_test_deps diff --git a/tests/integration/_support/openai_wire.py b/tests/integration/_support/openai_wire.py index d16cb60c455..bf9d5f194c1 100644 --- a/tests/integration/_support/openai_wire.py +++ b/tests/integration/_support/openai_wire.py @@ -7,8 +7,9 @@ from typing import Final from integration._support.wire import Reply, Request, Wire from pydantic import JsonValue +from tests.integration._support.provider import MODEL_DISCOVERY + _USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8} -MODEL_DISCOVERY: Final = ("GET", "/v1/models") def answering_model_discovery(respond: Callable[[Request], Reply]) -> Callable[[Request], Reply]: diff --git a/tests/integration/_support/provider.py b/tests/integration/_support/provider.py index 278cd054c61..80c14564e25 100644 --- a/tests/integration/_support/provider.py +++ b/tests/integration/_support/provider.py @@ -3,6 +3,7 @@ Deployments in `proxy_config.yaml` point at `PROVIDER_URL`, so one server answers for all of them. A test queues the replies it expects with `expect` and reads what the proxy sent with `received`. Tests run one at a time against it; the `provider` fixture checks nothing is left over between tests. +The proxy's model-info refresh is answered here. """ from __future__ import annotations @@ -17,6 +18,7 @@ from tests.integration._support.wire import Reply, Request, Wire, wire_server PROVIDER_PORT: Final = 8191 PROVIDER_URL: Final = f"http://127.0.0.1:{PROVIDER_PORT}" +MODEL_DISCOVERY: Final = ("GET", "/v1/models") _UNQUEUED: Final = Reply(status=500, body=b'{"error": "the shared fake provider has no reply queued for this request"}') @@ -30,7 +32,9 @@ class SharedProvider: self.replies.extend(replies) def received(self) -> tuple[Request, ...]: - return self.wire.drain() + return tuple( + request for request in self.wire.drain() if (request.method, request.target) != MODEL_DISCOVERY + ) @contextmanager @@ -38,6 +42,8 @@ def shared_provider() -> Iterator[SharedProvider]: replies: Final[deque[Reply]] = deque() def respond(request: Request) -> Reply: + if (request.method, request.target) == MODEL_DISCOVERY: + return Reply(body=b'{"object":"list","data":[]}') return replies.popleft() if replies else _UNQUEUED with wire_server(respond, port=PROVIDER_PORT) as wire: diff --git a/tests/integration/compatibility/test_codex_model_catalog.py b/tests/integration/compatibility/test_codex_model_catalog.py index f9bcc3b6f73..f1eea96a268 100644 --- a/tests/integration/compatibility/test_codex_model_catalog.py +++ b/tests/integration/compatibility/test_codex_model_catalog.py @@ -3,21 +3,33 @@ import json import os import time import uuid -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from hashlib import sha256 from itertools import chain from pathlib import Path +from types import MappingProxyType from typing import Final, TypeVar import anthropic import httpx import openai +import pytest +import yaml from pydantic import JsonValue from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS -from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy from tests.integration._support.wire import Reply, Request, Wire, wire_server T = TypeVar("T") @@ -36,6 +48,25 @@ ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"} USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15} +def _config_without_model_list(directory: Path) -> Path: + shared_config: Final = object_value(yaml.safe_load((ROOT / "tests/integration/proxy_config.yaml").read_text())) + config_without_models: Final[Mapping[str, JsonValue]] = MappingProxyType( + {name: value for name, value in shared_config.items() if name != "model_list"} + ) + path: Final = directory / "proxy_config_without_model_list.yaml" + path.write_text(yaml.safe_dump({**config_without_models})) + return path + + +@pytest.fixture(scope="module") +def gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + with gateway_from_environment() as parent_gateway: + directory: Final = tmp_path_factory.mktemp("codex_model_catalog") + config: Final = _config_without_model_list(directory) + with owned_proxy(parent_gateway, directory, {}, config=config, workers=PROXY_WORKERS) as isolated_gateway: + yield isolated_gateway + + def _generic_tier(identity: str) -> dict[str, JsonValue]: return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"} diff --git a/tests/litellm_utils_tests/test_secret_manager.py b/tests/litellm_utils_tests/test_secret_manager.py index ae2b55c7b0d..7febb7aea39 100644 --- a/tests/litellm_utils_tests/test_secret_manager.py +++ b/tests/litellm_utils_tests/test_secret_manager.py @@ -1,4 +1,5 @@ import base64 +import hashlib import os import time import traceback @@ -10,6 +11,7 @@ import json load_dotenv() import tempfile from uuid import uuid4 +from typing import Final import pytest import litellm @@ -23,6 +25,8 @@ from litellm.secret_managers.main import ( ) from unittest.mock import AsyncMock, patch, MagicMock +_AWS_FIXTURE_MASTER_KEY_SHA256: Final = "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b" + def load_vertex_ai_credentials(): # Define the path to the vertex_key.json file @@ -76,7 +80,9 @@ def test_aws_secret_manager(): # cast json to dict secret_val = json.loads(secret_val) - assert secret_val["litellm_master_key"] == os.environ["LITELLM_MASTER_KEY"] + assert ( + hashlib.sha256(secret_val["litellm_master_key"].encode()).hexdigest() == _AWS_FIXTURE_MASTER_KEY_SHA256 + ), "Expected the fixture value stored in the CI AWS account's litellm_master_key secret" def redact_oidc_signature(secret_val): diff --git a/tests/mcp_tests/test_proxy_mcp_e2e.py b/tests/mcp_tests/test_proxy_mcp_e2e.py index 352f55eb983..d22d54f0bce 100644 --- a/tests/mcp_tests/test_proxy_mcp_e2e.py +++ b/tests/mcp_tests/test_proxy_mcp_e2e.py @@ -390,7 +390,7 @@ async def _http_streams(url: str, headers: dict[str, str]): @pytest.mark.asyncio async def test_unchanged_sdk1_langchain_peer_can_list_and_call(proxy_server_url: str) -> None: script = """ -import asyncio, json, sys +import asyncio, json, os, sys from mcp import ClientSession from mcp.client.streamable_http import streamablehttp_client from langchain_mcp_adapters.tools import load_mcp_tools @@ -706,7 +706,7 @@ class TestProxyMcpSchemaDiscoveryMode: async def authorize_proxy_key(request: Request, api_key: str) -> UserAPIKeyAuth: permissions = { "sk-schema": LiteLLM_ObjectPermissionTable(object_permission_id="schema", mcp_servers=["schema"]), - "sk-9876": LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]), + MASTER_KEY: LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]), "sk-restricted": LiteLLM_ObjectPermissionTable( object_permission_id="restricted", mcp_servers=["math_restricted"] ), @@ -746,7 +746,7 @@ proxy_call_recorder = ProxyCallRecorder() @asynccontextmanager -async def _scoped_session(url: str, key: str = "sk-9876", **headers: str) -> typing.AsyncIterator[ClientSession]: +async def _scoped_session(url: str, key: str = MASTER_KEY, **headers: str) -> typing.AsyncIterator[ClientSession]: async with asyncio.timeout(30): async with _proxy_session(url, Authorization=f"Bearer {key}", **headers) as (read, write): async with ClientSession(read, write) as session: diff --git a/tests/unified_google_tests/base_google_genai_proxy_sdk_test.py b/tests/unified_google_tests/base_google_genai_proxy_sdk_test.py index 328c188e1af..50d419c704b 100644 --- a/tests/unified_google_tests/base_google_genai_proxy_sdk_test.py +++ b/tests/unified_google_tests/base_google_genai_proxy_sdk_test.py @@ -6,6 +6,8 @@ from typing import Any, Dict, List, Optional import pytest +from tests._master_key import MASTER_KEY + try: from google import genai from google.genai import types @@ -14,7 +16,6 @@ try: except ImportError: GOOGLE_GENAI_SDK_AVAILABLE = False -MASTER_KEY = "sk-unified-google-tests-4f9b2c7d8e1a" PROMPT = "Reply with only the single word: pong"