From e68cf49445f6637f3581ffa44df096e1496fdbf2 Mon Sep 17 00:00:00 2001 From: yuneng Date: Thu, 24 Sep 2026 01:46:56 +0000 Subject: [PATCH] ci: keep project secrets out of litellm-tests and move key-dependent tests to their tiers --- .circleci/scripts/unit_selection.sh | 3 +- .circleci/tests.yml | 18 +- .../test_token_counter_gemini_contents_e2e.py | 78 +++++++++ tests/integration/mcp_sdk1/__init__.py | 0 tests/integration/mcp_sdk1/conftest.py | 53 ------ .../routing/test_user_config_routing.py | 103 ++++++++++++ .../mcp_server.py} | 0 .../test_configs}/test_config_mcp_e2e.yaml | 2 +- tests/mcp_tests/test_mcp_litellm_client.py | 4 +- .../test_proxy_mcp_e2e.py | 6 +- .../test_prometheus_unit_tests.py | 10 +- tests/unit/proxy/test_proxy_custom_auth.py | 5 +- .../unit/proxy/test_proxy_pass_user_config.py | 114 ------------- tests/unit/proxy/test_proxy_token_counter.py | 159 +++--------------- tests/unit/proxy/test_proxy_utils.py | 4 +- 15 files changed, 232 insertions(+), 327 deletions(-) create mode 100644 tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py delete mode 100644 tests/integration/mcp_sdk1/__init__.py delete mode 100644 tests/integration/mcp_sdk1/conftest.py create mode 100644 tests/integration/routing/test_user_config_routing.py rename tests/{integration/mcp_sdk1/mcp_math_server.py => mcp_tests/mcp_server.py} (100%) rename tests/{integration/mcp_sdk1 => mcp_tests/test_configs}/test_config_mcp_e2e.yaml (91%) rename tests/{integration/mcp_sdk1 => mcp_tests}/test_proxy_mcp_e2e.py (99%) delete mode 100644 tests/unit/proxy/test_proxy_pass_user_config.py diff --git a/.circleci/scripts/unit_selection.sh b/.circleci/scripts/unit_selection.sh index bac7fdad842..f2ee7550df3 100755 --- a/.circleci/scripts/unit_selection.sh +++ b/.circleci/scripts/unit_selection.sh @@ -50,7 +50,7 @@ legacy_paths() { mcp-integration) echo tests/unit/proxy/_experimental/mcp_server echo tests/unit/responses/mcp - echo tests/integration/mcp_sdk1 ;; + echo tests/mcp_tests/test_proxy_mcp_e2e.py ;; proxy-db-auth-checks) echo tests/unit/proxy/auth/test_auth_checks.py echo tests/unit/proxy/auth/test_user_api_key_auth.py @@ -105,7 +105,6 @@ legacy_paths() { echo tests/unit/proxy/auth/test_proxy_routes.py echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py echo tests/unit/proxy/test_proxy_config_unit_test.py - echo tests/unit/proxy/test_proxy_pass_user_config.py echo tests/unit/proxy/test_proxy_token_counter.py echo tests/unit/proxy/test_server_root_path.py ;; proxy-db-proxy-server-core) diff --git a/.circleci/tests.yml b/.circleci/tests.yml index 423a9addae6..f6c936cb695 100644 --- a/.circleci/tests.yml +++ b/.circleci/tests.yml @@ -186,6 +186,9 @@ jobs: legacy_mcp_peer: type: boolean default: false + reruns: + type: integer + default: 0 machine: image: ubuntu-2204:2024.04.1 resource_class: large @@ -214,14 +217,19 @@ jobs: no_output_timeout: 20m command: | mkdir -p test-results/<< parameters.flag >> - selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" + selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; } [ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; } mapfile -t files < <(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename) if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; fi xdist_args=() if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi + rerun_args=(-p no:rerunfailures) + if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi + test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP") + if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi set +e - uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml + env -i "${test_env[@]}" \ + uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml status=$? set -e if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi @@ -282,7 +290,7 @@ jobs: - start_redis - run: name: Run owned integration contracts - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> + command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >> no_output_timeout: 15m - run: name: Stop owned database and Redis @@ -309,6 +317,7 @@ workflows: name: unit-<< matrix.flag >> shards: 1 workers: 2 + reruns: 2 matrix: parameters: flag: [caching-local, proxy-extras, enterprise-routing] @@ -325,6 +334,7 @@ workflows: - unit: name: unit-<< matrix.flag >> shards: 1 + reruns: 2 matrix: parameters: flag: @@ -346,6 +356,7 @@ workflows: name: unit-proxy-db-proxy-utils flag: proxy-db-proxy-utils shards: 1 + reruns: 2 dist: worksteal base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> @@ -354,6 +365,7 @@ workflows: flag: proxy-db-key-generation shards: 1 workers: 0 + reruns: 2 base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >> pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >> - documentation diff --git a/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py new file mode 100644 index 00000000000..b2b8f46ab0a --- /dev/null +++ b/tests/e2e/llm_translation/test_token_counter_gemini_contents_e2e.py @@ -0,0 +1,78 @@ +"""Live e2e: `/utils/token_counter?call_endpoint=true` counts Gemini `contents` upstream. + +Google's countTokens API is the only tokenizer that knows Gemini's real token +boundaries, so the proxy must forward `contents` to it for both the AI Studio and +Vertex deployments and hand back the provider's `promptTokensDetails`. Claude on +Vertex is covered by `/v1/messages/count_tokens`; this is the Gemini `contents` +route the claude_code rows never reach +""" + +from __future__ import annotations + +import pytest +from e2e_config import unique_marker +from e2e_http import require_successful_call +from proxy_client import ProxyClient +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + +GEMINI_DEPLOYMENTS = ("gemini-2.5-flash", "gemini-2.5-flash-vertex") + + +class _Part(BaseModel): + text: str + + +class _Content(BaseModel): + parts: tuple[_Part, ...] + + +class _TokenCountBody(BaseModel): + model: str + contents: tuple[_Content, ...] + + +class _CallEndpoint(BaseModel): + call_endpoint: bool = True + + +class _ModalityTokens(BaseModel): + modality: str + tokenCount: int + + +class _CountTokensUpstream(BaseModel): + totalTokens: int + promptTokensDetails: tuple[_ModalityTokens, ...] + + +class _TokenCountResponse(BaseModel): + total_tokens: int + request_model: str + model_used: str + tokenizer_type: str + original_response: _CountTokensUpstream + + +class TestGeminiContentsTokenCounting: + @pytest.mark.parametrize("model", GEMINI_DEPLOYMENTS) + def test_contents_are_counted_by_the_provider_endpoint( + self, proxy: ProxyClient, scoped_key: str, model: str + ) -> None: + text = f"Hello world, how are you doing today? {unique_marker()}" + body = _TokenCountBody(model=model, contents=(_Content(parts=(_Part(text=text),)),)) + + result = proxy.transport.send( + "/utils/token_counter", + headers=proxy.transport.bearer(scoped_key), + json=body, + params=_CallEndpoint(), + ) + + require_successful_call(result) + counted = _TokenCountResponse.model_validate_json(result.body) + assert counted.request_model == model, counted + assert counted.original_response.totalTokens == counted.total_tokens > 0, counted + assert counted.original_response.promptTokensDetails, counted + assert all(detail.tokenCount > 0 for detail in counted.original_response.promptTokensDetails), counted diff --git a/tests/integration/mcp_sdk1/__init__.py b/tests/integration/mcp_sdk1/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/mcp_sdk1/conftest.py b/tests/integration/mcp_sdk1/conftest.py deleted file mode 100644 index 3b265d846ab..00000000000 --- a/tests/integration/mcp_sdk1/conftest.py +++ /dev/null @@ -1,53 +0,0 @@ -import asyncio -import importlib - -import pytest - -import litellm -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - -@pytest.fixture(scope="session") -def event_loop(): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - importlib.reload(litellm) - import asyncio - - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - yield - - # Teardown code (executes after the yield point) - # LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks - asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue()) - loop.close() # Close the loop created earlier - asyncio.set_event_loop(None) # Remove the reference to the loop - - -@pytest.fixture(scope="function", autouse=True) -async def drain_logging_worker(): - """ - The logging queue is bound to the running loop, so anything left queued when a test's loop - goes away is carried onto the next test's loop and fires against its callbacks. - """ - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - yield - - try: - await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10) - except asyncio.TimeoutError: - pass - diff --git a/tests/integration/routing/test_user_config_routing.py b/tests/integration/routing/test_user_config_routing.py new file mode 100644 index 00000000000..1c50a78201b --- /dev/null +++ b/tests/integration/routing/test_user_config_routing.py @@ -0,0 +1,103 @@ +import json +import uuid +from pathlib import Path +from typing import Final + +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex + + +def _completion(request: Request) -> Reply: + if request.target != "/v1/chat/completions": + return Reply(status=404, body=b"{}") + body: Final = json.loads(request.body) + return Reply( + body=json.dumps( + { + "id": "chatcmpl-user-config", + "object": "chat.completion", + "created": 0, + "model": body["model"], + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } + ).encode() + ) + + +def _user_config(upstream_url: str) -> dict[str, object]: + return { + "model_list": [ + { + "model_name": "user-config-deployment", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_base": upstream_url + "/v1", + "api_key": USER_KEY, + }, + } + ], + "num_retries": 0, + } + + +def _opt_in_config(directory: Path, upstream_url: str) -> Path: + config: Final = directory / "allow_client_side_credentials_config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "admin-deployment", + "litellm_params": { + "model": "openai/gpt-4.1-mini", + "api_base": upstream_url + "/v1", + "api_key": "sk-admin-configured", + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "allow_client_side_credentials": True, + }, + } + ) + ) + return config + + +def _request_body(upstream_url: str) -> dict[str, object]: + return { + "model": "user-config-deployment", + "messages": [{"role": "user", "content": "user config control"}], + "user_config": _user_config(upstream_url), + } + + +def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gateway: Gateway, tmp_path: Path) -> None: + with wire_server(_completion) as upstream: + config: Final = _opt_in_config(tmp_path, upstream.url) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + response: Final = candidate.request("POST", "/v1/chat/completions", _request_body(upstream.url)) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == "routed" + outbound: Final = tuple(upstream.received.get_nowait() for _ in range(upstream.received.qsize())) + completions: Final = tuple(request for request in outbound if request.target == "/v1/chat/completions") + assert len(completions) == 1, outbound + assert completions[0].headers["authorization"] == f"Bearer {USER_KEY}" + assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini" + + +def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None: + with wire_server(_completion) as upstream: + response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url)) + assert response.status_code == 401, response.text + assert "user_config is not allowed in request body" in response.text + assert upstream.received.empty() diff --git a/tests/integration/mcp_sdk1/mcp_math_server.py b/tests/mcp_tests/mcp_server.py similarity index 100% rename from tests/integration/mcp_sdk1/mcp_math_server.py rename to tests/mcp_tests/mcp_server.py diff --git a/tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml b/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml similarity index 91% rename from tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml rename to tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml index 29cb8d2726d..19fad3d1393 100644 --- a/tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml +++ b/tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml @@ -17,7 +17,7 @@ mcp_servers: transport: stdio command: python3 args: - - tests/integration/mcp_sdk1/mcp_math_server.py + - tests/mcp_tests/mcp_server.py allow_all_keys: true # Needed for master_key auth when DB/object_permission not used math_streamable_http: transport: http diff --git a/tests/mcp_tests/test_mcp_litellm_client.py b/tests/mcp_tests/test_mcp_litellm_client.py index caabe18adb0..cfc0692c8fa 100644 --- a/tests/mcp_tests/test_mcp_litellm_client.py +++ b/tests/mcp_tests/test_mcp_litellm_client.py @@ -18,8 +18,8 @@ import json async def test_mcp_agent(): """Test MCP agent functionality with a simple math server""" try: - local_server_path = "./mcp_math_server.py" - ci_cd_server_path = "tests/integration/mcp_sdk1/mcp_math_server.py" + local_server_path = "./mcp_server.py" + ci_cd_server_path = "tests/mcp_tests/mcp_server.py" # Use the correct path for the server server_path = ( diff --git a/tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py b/tests/mcp_tests/test_proxy_mcp_e2e.py similarity index 99% rename from tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py rename to tests/mcp_tests/test_proxy_mcp_e2e.py index 8fb302855f5..a730f6c10ee 100644 --- a/tests/integration/mcp_sdk1/test_proxy_mcp_e2e.py +++ b/tests/mcp_tests/test_proxy_mcp_e2e.py @@ -35,10 +35,10 @@ from litellm.proxy.proxy_server import ( initialize, ) -PROJECT_ROOT = Path(__file__).resolve().parents[3] -CONFIG_TEMPLATE_PATH = PROJECT_ROOT / "tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml" -MCP_SERVER_SCRIPT = PROJECT_ROOT / "tests/integration/mcp_sdk1/mcp_math_server.py" +CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml") +MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py") MCP_PEER_PYTHON = os.environ.get("MCP_TEST_PEER_PYTHON", sys.executable) +PROJECT_ROOT = Path(__file__).resolve().parents[2] PROXY_START_TIMEOUT = 30 diff --git a/tests/unit/enterprise/integrations/test_prometheus_unit_tests.py b/tests/unit/enterprise/integrations/test_prometheus_unit_tests.py index 28fd03daf37..5b26dad269e 100644 --- a/tests/unit/enterprise/integrations/test_prometheus_unit_tests.py +++ b/tests/unit/enterprise/integrations/test_prometheus_unit_tests.py @@ -13,7 +13,6 @@ import asyncio from dotenv import load_dotenv load_dotenv() -import os from unittest.mock import MagicMock @@ -165,9 +164,9 @@ async def test_prometheus_metric_tracking(): "model_name": "gpt-5-mini", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_version": os.getenv("AZURE_AI_API_VERSION"), - "api_base": os.getenv("AZURE_AI_API_BASE"), + "api_key": "sk-azure-unit-test", + "api_version": "2025-01-01-preview", + "api_base": "https://unit-test.openai.azure.com", }, "model_info": {"id": "azure-model-id"}, }, @@ -180,9 +179,6 @@ async def test_prometheus_metric_tracking(): }, ], provider_budget_config=provider_budget_config, - redis_host=os.getenv("REDIS_HOST"), - redis_port=int(os.getenv("REDIS_PORT", 6379)), - redis_password=os.getenv("REDIS_PASSWORD"), ) try: diff --git a/tests/unit/proxy/test_proxy_custom_auth.py b/tests/unit/proxy/test_proxy_custom_auth.py index b575e4c85c6..dbbad0dab1e 100644 --- a/tests/unit/proxy/test_proxy_custom_auth.py +++ b/tests/unit/proxy/test_proxy_custom_auth.py @@ -53,8 +53,7 @@ def test_custom_auth(client): "max_tokens": 10, } # Your bearer token - token = os.getenv("PROXY_MASTER_KEY") - print(f"token: {token}") + token = "sk-unit-test-master" headers = {"Authorization": f"Bearer {token}"} with pytest.raises(Exception, match="Authentication Error, Failed custom auth") as exc_info: client.post("/chat/completions", json=test_data, headers=headers) @@ -71,7 +70,7 @@ def test_custom_auth_bearer(client): "max_tokens": 10, } # Your bearer token - token = os.getenv("PROXY_MASTER_KEY") + token = "sk-unit-test-master" headers = {"Authorization": f"WITHOUT BEAR Er {token}"} with pytest.raises(Exception, match="CustomAuth - Malformed API Key passed in") as exc_info: diff --git a/tests/unit/proxy/test_proxy_pass_user_config.py b/tests/unit/proxy/test_proxy_pass_user_config.py deleted file mode 100644 index 91911c142ea..00000000000 --- a/tests/unit/proxy/test_proxy_pass_user_config.py +++ /dev/null @@ -1,114 +0,0 @@ -import sys, os -import traceback -from dotenv import load_dotenv - -load_dotenv() -import io - -# this file is to test litellm/proxy - -import pytest, logging, asyncio -import litellm -from litellm import embedding, completion, completion_cost, Timeout -from litellm import RateLimitError - -# Configure logging -logging.basicConfig( - level=logging.DEBUG, # Set the desired logging level - format="%(asctime)s - %(levelname)s - %(message)s", -) - -# test /chat/completion request to the proxy -from fastapi.testclient import TestClient -from fastapi import FastAPI -from litellm.proxy.proxy_server import ( - router, - save_worker_config, - initialize, -) # Replace with the actual module where your FastAPI router is defined - -# Your bearer token -token = "sk-1234" - -headers = {"Authorization": f"Bearer {token}"} - - -@pytest.fixture(scope="function") -def client_no_auth(): - # Assuming litellm.proxy.proxy_server is an object - from litellm.proxy.proxy_server import cleanup_router_config_variables - - cleanup_router_config_variables() - filepath = os.path.dirname(os.path.abspath(__file__)) - config_fp = f"{filepath}/test_configs/test_config_no_auth.yaml" - # initialize can get run in parallel, it sets specific variables for the fast api app, sinc eit gets run in parallel different tests use the wrong variables - asyncio.run(initialize(config=config_fp, debug=True)) - app = FastAPI() - app.include_router(router) # Include your router in the test app - - return TestClient(app) - - -@pytest.mark.skipif( - os.environ.get("AZURE_AI_API_KEY") is None - or os.environ.get("OPENAI_API_KEY") is None, - reason="AZURE_AI_API_KEY or OPENAI_API_KEY not set - skipping integration test", -) -def test_chat_completion(client_no_auth): - global headers - - from litellm.types.router import RouterConfig, ModelConfig - from litellm.types.completion import CompletionRequest - - user_config = RouterConfig( - model_list=[ - ModelConfig( - model_name="user-azure-instance", - litellm_params=CompletionRequest( - model="azure/gpt-4.1-mini", - api_key=os.getenv("AZURE_AI_API_KEY"), - api_version=os.getenv("AZURE_API_VERSION"), - api_base=os.getenv("AZURE_AI_API_BASE"), - timeout=10, - ), - tpm=240000, - rpm=1800, - ), - ModelConfig( - model_name="user-openai-instance", - litellm_params=CompletionRequest( - model="gpt-3.5-turbo", - api_key=os.getenv("OPENAI_API_KEY"), - timeout=10, - ), - tpm=240000, - rpm=1800, - ), - ], - num_retries=2, - allowed_fails=3, - fallbacks=[{"user-azure-instance": ["user-openai-instance"]}], - ).dict() - - try: - # Your test data - test_data = { - "model": "user-azure-instance", - "messages": [ - {"role": "user", "content": "hi"}, - ], - "max_tokens": 10, - "user_config": user_config, - } - - print("testing proxy server with chat completions") - response = client_no_auth.post("/v1/chat/completions", json=test_data) - print(f"response - {response.text}") - assert response.status_code == 200 - result = response.json() - print(f"Received response: {result}") - except Exception as e: - pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}") - - -# Run the test diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 39ec4bb1887..8590e959961 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2,10 +2,7 @@ # 1. Generate a Key, and use it to make a call -import json import logging -import os -import tempfile from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -35,79 +32,6 @@ from litellm.types.utils import TokenCountResponse verbose_proxy_logger.setLevel(level=logging.DEBUG) -def get_vertex_ai_creds_json() -> dict: - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - return service_account_key_data - - -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) - - @pytest.mark.asyncio async def test_vLLM_token_counting(): """ @@ -223,10 +147,12 @@ async def test_anthropic_messages_count_tokens_endpoint(): - Should return response in Anthropic format: {"input_tokens": } - Should work as wrapper around internal token_counter function """ - from litellm.proxy.anthropic_endpoints.endpoints import count_tokens - from fastapi import Request from unittest.mock import MagicMock + from fastapi import Request + + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + # Mock request object mock_request = MagicMock(spec=Request) mock_request_data = { @@ -295,10 +221,12 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): - Should still work and return Anthropic format - Should call internal token_counter with from_anthropic_endpoint=True """ - from litellm.proxy.anthropic_endpoints.endpoints import count_tokens - from fastapi import Request from unittest.mock import MagicMock + from fastapi import Request + + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + # Mock request object mock_request = MagicMock(spec=Request) mock_request_data = { @@ -435,10 +363,12 @@ async def test_anthropic_endpoint_error_handling(): """ Test error handling in the /v1/messages/count_tokens endpoint """ - from litellm.proxy.anthropic_endpoints.endpoints import count_tokens - from fastapi import Request, HTTPException from unittest.mock import MagicMock + from fastapi import HTTPException, Request + + from litellm.proxy.anthropic_endpoints.endpoints import count_tokens + # Mock request object mock_request = MagicMock(spec=Request) mock_user_api_key_dict = MagicMock() @@ -474,8 +404,10 @@ async def test_anthropic_endpoint_error_handling(): @pytest.mark.asyncio async def test_factory_anthropic_endpoint_calls_anthropic_counter(): """Test that /v1/messages/count_tokens with Anthropic model uses Anthropic counter.""" - from unittest.mock import patch, AsyncMock, MagicMock + from unittest.mock import AsyncMock, MagicMock, patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app # Mock the global handler instance in token_counter module @@ -531,8 +463,10 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter(): @pytest.mark.asyncio async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter(): """Test that /v1/messages/count_tokens with GPT-4 does NOT use Anthropic counter.""" - from unittest.mock import patch, AsyncMock, MagicMock + from unittest.mock import AsyncMock, MagicMock, patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app # Mock the global handler instance in token_counter module @@ -590,8 +524,10 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter(): @pytest.mark.asyncio async def test_factory_normal_token_counter_endpoint_does_not_call_anthropic(): """Test that /utils/token_counter does NOT use Anthropic counter even with Anthropic model.""" - from unittest.mock import patch, AsyncMock, MagicMock + from unittest.mock import AsyncMock, MagicMock, patch + from fastapi.testclient import TestClient + from litellm.proxy.proxy_server import app # Mock the global handler instance in token_counter module @@ -678,57 +614,6 @@ async def test_factory_registration(): assert not counter.should_use_token_counting_api(custom_llm_provider=None) -@pytest.mark.skip( - reason="Requires Google/Vertex AI credentials (GEMINI_API_KEY or VERTEX_AI_PRIVATE_KEY)." -) -@pytest.mark.asyncio -@pytest.mark.parametrize("model_name", ["gemini-2.5-pro", "vertex-ai-gemini-2.5-pro"]) -async def test_vertex_ai_gemini_token_counting_with_contents(model_name): - """ - Test token counting for Vertex AI Gemini model using contents format with call_endpoint=True - """ - load_vertex_ai_credentials() - llm_router = Router( - model_list=[ - { - "model_name": "gemini-2.5-pro", - "litellm_params": { - "model": "gemini/gemini-2.5-pro", - }, - }, - { - "model_name": "vertex-ai-gemini-2.5-pro", - "litellm_params": { - "model": "vertex_ai/gemini-2.5-pro", - }, - }, - ] - ) - - setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - - # Test with contents format and call_endpoint=True - response = await token_counter( - request=TokenCountRequest( - model=model_name, - contents=[ - {"parts": [{"text": "Hello world, how are you doing today? i am ij"}]} - ], - ), - call_endpoint=True, - ) - - print("Vertex AI Gemini token counting response:", response) - - # validate we have original response - assert response.original_response is not None - assert response.original_response.get("totalTokens") is not None - assert response.original_response.get("promptTokensDetails") is not None - - prompt_tokens_details = response.original_response.get("promptTokensDetails") - assert prompt_tokens_details is not None - - @pytest.mark.asyncio async def test_bedrock_count_tokens_endpoint(): """ @@ -779,7 +664,7 @@ async def test_vertex_ai_anthropic_token_counting(): This tests the token counting implementation for Vertex AI partner models without making actual API calls. Mocks at the handler level to test the full flow. """ - from unittest.mock import AsyncMock, patch, MagicMock + from unittest.mock import patch # Mock the Vertex AI partner models token counter response mock_token_response = { diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index 1134f41a940..ab787bbbe27 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -438,7 +438,7 @@ def test_is_request_body_safe_global_enabled( "model_name": "gpt-3.5-turbo", "litellm_params": { "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), + "api_key": "sk-openai-unit-test", }, } ] @@ -475,7 +475,7 @@ def test_is_request_body_safe_model_enabled( "model_name": "fireworks_ai/*", "litellm_params": { "model": "fireworks_ai/*", - "api_key": os.getenv("FIREWORKS_API_KEY"), + "api_key": "sk-fireworks-unit-test", "configurable_clientside_auth_params": ( ["api_base"] if allow_client_side_credentials else [] ),