ci: keep project secrets out of litellm-tests and move key-dependent tests to their tiers

This commit is contained in:
yuneng 2026-09-24 01:46:56 +00:00
parent 8376a5a431
commit e68cf49445
15 changed files with 232 additions and 327 deletions

View file

@ -50,7 +50,7 @@ legacy_paths() {
mcp-integration)
echo tests/unit/proxy/_experimental/mcp_server
echo tests/unit/responses/mcp
echo tests/integration/mcp_sdk1 ;;
echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
proxy-db-auth-checks)
echo tests/unit/proxy/auth/test_auth_checks.py
echo tests/unit/proxy/auth/test_user_api_key_auth.py
@ -105,7 +105,6 @@ legacy_paths() {
echo tests/unit/proxy/auth/test_proxy_routes.py
echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
echo tests/unit/proxy/test_proxy_config_unit_test.py
echo tests/unit/proxy/test_proxy_pass_user_config.py
echo tests/unit/proxy/test_proxy_token_counter.py
echo tests/unit/proxy/test_server_root_path.py ;;
proxy-db-proxy-server-core)

View file

@ -186,6 +186,9 @@ jobs:
legacy_mcp_peer:
type: boolean
default: false
reruns:
type: integer
default: 0
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
@ -214,14 +217,19 @@ jobs:
no_output_timeout: 20m
command: |
mkdir -p test-results/<< parameters.flag >>
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)"
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
[ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
mapfile -t files < <(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)
if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; fi
xdist_args=()
if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
rerun_args=(-p no:rerunfailures)
if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
set +e
uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
env -i "${test_env[@]}" \
uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
status=$?
set -e
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
@ -282,7 +290,7 @@ jobs:
- start_redis
- run:
name: Run owned integration contracts
command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
no_output_timeout: 15m
- run:
name: Stop owned database and Redis
@ -309,6 +317,7 @@ workflows:
name: unit-<< matrix.flag >>
shards: 1
workers: 2
reruns: 2
matrix:
parameters:
flag: [caching-local, proxy-extras, enterprise-routing]
@ -325,6 +334,7 @@ workflows:
- unit:
name: unit-<< matrix.flag >>
shards: 1
reruns: 2
matrix:
parameters:
flag:
@ -346,6 +356,7 @@ workflows:
name: unit-proxy-db-proxy-utils
flag: proxy-db-proxy-utils
shards: 1
reruns: 2
dist: worksteal
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
@ -354,6 +365,7 @@ workflows:
flag: proxy-db-key-generation
shards: 1
workers: 0
reruns: 2
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- documentation

View file

@ -0,0 +1,78 @@
"""Live e2e: `/utils/token_counter?call_endpoint=true` counts Gemini `contents` upstream.
Google's countTokens API is the only tokenizer that knows Gemini's real token
boundaries, so the proxy must forward `contents` to it for both the AI Studio and
Vertex deployments and hand back the provider's `promptTokensDetails`. Claude on
Vertex is covered by `/v1/messages/count_tokens`; this is the Gemini `contents`
route the claude_code rows never reach
"""
from __future__ import annotations
import pytest
from e2e_config import unique_marker
from e2e_http import require_successful_call
from proxy_client import ProxyClient
from pydantic import BaseModel
pytestmark = pytest.mark.e2e
GEMINI_DEPLOYMENTS = ("gemini-2.5-flash", "gemini-2.5-flash-vertex")
class _Part(BaseModel):
text: str
class _Content(BaseModel):
parts: tuple[_Part, ...]
class _TokenCountBody(BaseModel):
model: str
contents: tuple[_Content, ...]
class _CallEndpoint(BaseModel):
call_endpoint: bool = True
class _ModalityTokens(BaseModel):
modality: str
tokenCount: int
class _CountTokensUpstream(BaseModel):
totalTokens: int
promptTokensDetails: tuple[_ModalityTokens, ...]
class _TokenCountResponse(BaseModel):
total_tokens: int
request_model: str
model_used: str
tokenizer_type: str
original_response: _CountTokensUpstream
class TestGeminiContentsTokenCounting:
@pytest.mark.parametrize("model", GEMINI_DEPLOYMENTS)
def test_contents_are_counted_by_the_provider_endpoint(
self, proxy: ProxyClient, scoped_key: str, model: str
) -> None:
text = f"Hello world, how are you doing today? {unique_marker()}"
body = _TokenCountBody(model=model, contents=(_Content(parts=(_Part(text=text),)),))
result = proxy.transport.send(
"/utils/token_counter",
headers=proxy.transport.bearer(scoped_key),
json=body,
params=_CallEndpoint(),
)
require_successful_call(result)
counted = _TokenCountResponse.model_validate_json(result.body)
assert counted.request_model == model, counted
assert counted.original_response.totalTokens == counted.total_tokens > 0, counted
assert counted.original_response.promptTokensDetails, counted
assert all(detail.tokenCount > 0 for detail in counted.original_response.promptTokensDetails), counted

View file

@ -1,53 +0,0 @@
import asyncio
import importlib
import pytest
import litellm
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
@pytest.fixture(scope="session")
def event_loop():
try:
loop = asyncio.get_running_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
yield loop
loop.close()
@pytest.fixture(scope="function", autouse=True)
def setup_and_teardown():
"""
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
"""
importlib.reload(litellm)
import asyncio
loop = asyncio.get_event_loop_policy().new_event_loop()
asyncio.set_event_loop(loop)
yield
# Teardown code (executes after the yield point)
# LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks
asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue())
loop.close() # Close the loop created earlier
asyncio.set_event_loop(None) # Remove the reference to the loop
@pytest.fixture(scope="function", autouse=True)
async def drain_logging_worker():
"""
The logging queue is bound to the running loop, so anything left queued when a test's loop
goes away is carried onto the next test's loop and fires against its callbacks.
"""
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
yield
try:
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
except asyncio.TimeoutError:
pass

View file

@ -0,0 +1,103 @@
import json
import uuid
from pathlib import Path
from typing import Final
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex
def _completion(request: Request) -> Reply:
if request.target != "/v1/chat/completions":
return Reply(status=404, body=b"{}")
body: Final = json.loads(request.body)
return Reply(
body=json.dumps(
{
"id": "chatcmpl-user-config",
"object": "chat.completion",
"created": 0,
"model": body["model"],
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
}
).encode()
)
def _user_config(upstream_url: str) -> dict[str, object]:
return {
"model_list": [
{
"model_name": "user-config-deployment",
"litellm_params": {
"model": "openai/gpt-4.1-mini",
"api_base": upstream_url + "/v1",
"api_key": USER_KEY,
},
}
],
"num_retries": 0,
}
def _opt_in_config(directory: Path, upstream_url: str) -> Path:
config: Final = directory / "allow_client_side_credentials_config.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "admin-deployment",
"litellm_params": {
"model": "openai/gpt-4.1-mini",
"api_base": upstream_url + "/v1",
"api_key": "sk-admin-configured",
},
}
],
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
"store_model_in_db": True,
"allow_client_side_credentials": True,
},
}
)
)
return config
def _request_body(upstream_url: str) -> dict[str, object]:
return {
"model": "user-config-deployment",
"messages": [{"role": "user", "content": "user config control"}],
"user_config": _user_config(upstream_url),
}
def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gateway: Gateway, tmp_path: Path) -> None:
with wire_server(_completion) as upstream:
config: Final = _opt_in_config(tmp_path, upstream.url)
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
response: Final = candidate.request("POST", "/v1/chat/completions", _request_body(upstream.url))
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == "routed"
outbound: Final = tuple(upstream.received.get_nowait() for _ in range(upstream.received.qsize()))
completions: Final = tuple(request for request in outbound if request.target == "/v1/chat/completions")
assert len(completions) == 1, outbound
assert completions[0].headers["authorization"] == f"Bearer {USER_KEY}"
assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini"
def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None:
with wire_server(_completion) as upstream:
response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url))
assert response.status_code == 401, response.text
assert "user_config is not allowed in request body" in response.text
assert upstream.received.empty()

View file

@ -17,7 +17,7 @@ mcp_servers:
transport: stdio
command: python3
args:
- tests/integration/mcp_sdk1/mcp_math_server.py
- tests/mcp_tests/mcp_server.py
allow_all_keys: true # Needed for master_key auth when DB/object_permission not used
math_streamable_http:
transport: http

View file

@ -18,8 +18,8 @@ import json
async def test_mcp_agent():
"""Test MCP agent functionality with a simple math server"""
try:
local_server_path = "./mcp_math_server.py"
ci_cd_server_path = "tests/integration/mcp_sdk1/mcp_math_server.py"
local_server_path = "./mcp_server.py"
ci_cd_server_path = "tests/mcp_tests/mcp_server.py"
# Use the correct path for the server
server_path = (

View file

@ -35,10 +35,10 @@ from litellm.proxy.proxy_server import (
initialize,
)
PROJECT_ROOT = Path(__file__).resolve().parents[3]
CONFIG_TEMPLATE_PATH = PROJECT_ROOT / "tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml"
MCP_SERVER_SCRIPT = PROJECT_ROOT / "tests/integration/mcp_sdk1/mcp_math_server.py"
CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml")
MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py")
MCP_PEER_PYTHON = os.environ.get("MCP_TEST_PEER_PYTHON", sys.executable)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
PROXY_START_TIMEOUT = 30

View file

@ -13,7 +13,6 @@ import asyncio
from dotenv import load_dotenv
load_dotenv()
import os
from unittest.mock import MagicMock
@ -165,9 +164,9 @@ async def test_prometheus_metric_tracking():
"model_name": "gpt-5-mini", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/gpt-4.1-mini",
"api_key": os.getenv("AZURE_AI_API_KEY"),
"api_version": os.getenv("AZURE_AI_API_VERSION"),
"api_base": os.getenv("AZURE_AI_API_BASE"),
"api_key": "sk-azure-unit-test",
"api_version": "2025-01-01-preview",
"api_base": "https://unit-test.openai.azure.com",
},
"model_info": {"id": "azure-model-id"},
},
@ -180,9 +179,6 @@ async def test_prometheus_metric_tracking():
},
],
provider_budget_config=provider_budget_config,
redis_host=os.getenv("REDIS_HOST"),
redis_port=int(os.getenv("REDIS_PORT", 6379)),
redis_password=os.getenv("REDIS_PASSWORD"),
)
try:

View file

@ -53,8 +53,7 @@ def test_custom_auth(client):
"max_tokens": 10,
}
# Your bearer token
token = os.getenv("PROXY_MASTER_KEY")
print(f"token: {token}")
token = "sk-unit-test-master"
headers = {"Authorization": f"Bearer {token}"}
with pytest.raises(Exception, match="Authentication Error, Failed custom auth") as exc_info:
client.post("/chat/completions", json=test_data, headers=headers)
@ -71,7 +70,7 @@ def test_custom_auth_bearer(client):
"max_tokens": 10,
}
# Your bearer token
token = os.getenv("PROXY_MASTER_KEY")
token = "sk-unit-test-master"
headers = {"Authorization": f"WITHOUT BEAR Er {token}"}
with pytest.raises(Exception, match="CustomAuth - Malformed API Key passed in") as exc_info:

View file

@ -1,114 +0,0 @@
import sys, os
import traceback
from dotenv import load_dotenv
load_dotenv()
import io
# this file is to test litellm/proxy
import pytest, logging, asyncio
import litellm
from litellm import embedding, completion, completion_cost, Timeout
from litellm import RateLimitError
# Configure logging
logging.basicConfig(
level=logging.DEBUG, # Set the desired logging level
format="%(asctime)s - %(levelname)s - %(message)s",
)
# test /chat/completion request to the proxy
from fastapi.testclient import TestClient
from fastapi import FastAPI
from litellm.proxy.proxy_server import (
router,
save_worker_config,
initialize,
) # Replace with the actual module where your FastAPI router is defined
# Your bearer token
token = "sk-1234"
headers = {"Authorization": f"Bearer {token}"}
@pytest.fixture(scope="function")
def client_no_auth():
# Assuming litellm.proxy.proxy_server is an object
from litellm.proxy.proxy_server import cleanup_router_config_variables
cleanup_router_config_variables()
filepath = os.path.dirname(os.path.abspath(__file__))
config_fp = f"{filepath}/test_configs/test_config_no_auth.yaml"
# initialize can get run in parallel, it sets specific variables for the fast api app, sinc eit gets run in parallel different tests use the wrong variables
asyncio.run(initialize(config=config_fp, debug=True))
app = FastAPI()
app.include_router(router) # Include your router in the test app
return TestClient(app)
@pytest.mark.skipif(
os.environ.get("AZURE_AI_API_KEY") is None
or os.environ.get("OPENAI_API_KEY") is None,
reason="AZURE_AI_API_KEY or OPENAI_API_KEY not set - skipping integration test",
)
def test_chat_completion(client_no_auth):
global headers
from litellm.types.router import RouterConfig, ModelConfig
from litellm.types.completion import CompletionRequest
user_config = RouterConfig(
model_list=[
ModelConfig(
model_name="user-azure-instance",
litellm_params=CompletionRequest(
model="azure/gpt-4.1-mini",
api_key=os.getenv("AZURE_AI_API_KEY"),
api_version=os.getenv("AZURE_API_VERSION"),
api_base=os.getenv("AZURE_AI_API_BASE"),
timeout=10,
),
tpm=240000,
rpm=1800,
),
ModelConfig(
model_name="user-openai-instance",
litellm_params=CompletionRequest(
model="gpt-3.5-turbo",
api_key=os.getenv("OPENAI_API_KEY"),
timeout=10,
),
tpm=240000,
rpm=1800,
),
],
num_retries=2,
allowed_fails=3,
fallbacks=[{"user-azure-instance": ["user-openai-instance"]}],
).dict()
try:
# Your test data
test_data = {
"model": "user-azure-instance",
"messages": [
{"role": "user", "content": "hi"},
],
"max_tokens": 10,
"user_config": user_config,
}
print("testing proxy server with chat completions")
response = client_no_auth.post("/v1/chat/completions", json=test_data)
print(f"response - {response.text}")
assert response.status_code == 200
result = response.json()
print(f"Received response: {result}")
except Exception as e:
pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}")
# Run the test

View file

@ -2,10 +2,7 @@
# 1. Generate a Key, and use it to make a call
import json
import logging
import os
import tempfile
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -35,79 +32,6 @@ from litellm.types.utils import TokenCountResponse
verbose_proxy_logger.setLevel(level=logging.DEBUG)
def get_vertex_ai_creds_json() -> dict:
# Define the path to the vertex_key.json file
print("loading vertex ai credentials")
filepath = os.path.dirname(os.path.abspath(__file__))
vertex_key_path = filepath + "/vertex_key.json"
# Read the existing content of the file or create an empty dictionary
try:
with open(vertex_key_path, "r") as file:
# Read the file content
print("Read vertexai file path")
content = file.read()
# If the file is empty or not valid JSON, create an empty dictionary
if not content or not content.strip():
service_account_key_data = {}
else:
# Attempt to load the existing JSON content
file.seek(0)
service_account_key_data = json.load(file)
except FileNotFoundError:
# If the file doesn't exist, create an empty dictionary
service_account_key_data = {}
# Update the service_account_key_data with environment variables
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
private_key = private_key.replace("\\n", "\n")
service_account_key_data["private_key_id"] = private_key_id
service_account_key_data["private_key"] = private_key
return service_account_key_data
def load_vertex_ai_credentials():
# Define the path to the vertex_key.json file
print("loading vertex ai credentials")
filepath = os.path.dirname(os.path.abspath(__file__))
vertex_key_path = filepath + "/vertex_key.json"
# Read the existing content of the file or create an empty dictionary
try:
with open(vertex_key_path, "r") as file:
# Read the file content
print("Read vertexai file path")
content = file.read()
# If the file is empty or not valid JSON, create an empty dictionary
if not content or not content.strip():
service_account_key_data = {}
else:
# Attempt to load the existing JSON content
file.seek(0)
service_account_key_data = json.load(file)
except FileNotFoundError:
# If the file doesn't exist, create an empty dictionary
service_account_key_data = {}
# Update the service_account_key_data with environment variables
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
private_key = private_key.replace("\\n", "\n")
service_account_key_data["private_key_id"] = private_key_id
service_account_key_data["private_key"] = private_key
# Create a temporary file
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
# Write the updated content to the temporary files
json.dump(service_account_key_data, temp_file, indent=2)
# Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
@pytest.mark.asyncio
async def test_vLLM_token_counting():
"""
@ -223,10 +147,12 @@ async def test_anthropic_messages_count_tokens_endpoint():
- Should return response in Anthropic format: {"input_tokens": <count>}
- Should work as wrapper around internal token_counter function
"""
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
from fastapi import Request
from unittest.mock import MagicMock
from fastapi import Request
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
# Mock request object
mock_request = MagicMock(spec=Request)
mock_request_data = {
@ -295,10 +221,12 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
- Should still work and return Anthropic format
- Should call internal token_counter with from_anthropic_endpoint=True
"""
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
from fastapi import Request
from unittest.mock import MagicMock
from fastapi import Request
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
# Mock request object
mock_request = MagicMock(spec=Request)
mock_request_data = {
@ -435,10 +363,12 @@ async def test_anthropic_endpoint_error_handling():
"""
Test error handling in the /v1/messages/count_tokens endpoint
"""
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
from fastapi import Request, HTTPException
from unittest.mock import MagicMock
from fastapi import HTTPException, Request
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
# Mock request object
mock_request = MagicMock(spec=Request)
mock_user_api_key_dict = MagicMock()
@ -474,8 +404,10 @@ async def test_anthropic_endpoint_error_handling():
@pytest.mark.asyncio
async def test_factory_anthropic_endpoint_calls_anthropic_counter():
"""Test that /v1/messages/count_tokens with Anthropic model uses Anthropic counter."""
from unittest.mock import patch, AsyncMock, MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
from fastapi.testclient import TestClient
from litellm.proxy.proxy_server import app
# Mock the global handler instance in token_counter module
@ -531,8 +463,10 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter():
@pytest.mark.asyncio
async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
"""Test that /v1/messages/count_tokens with GPT-4 does NOT use Anthropic counter."""
from unittest.mock import patch, AsyncMock, MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
from fastapi.testclient import TestClient
from litellm.proxy.proxy_server import app
# Mock the global handler instance in token_counter module
@ -590,8 +524,10 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
@pytest.mark.asyncio
async def test_factory_normal_token_counter_endpoint_does_not_call_anthropic():
"""Test that /utils/token_counter does NOT use Anthropic counter even with Anthropic model."""
from unittest.mock import patch, AsyncMock, MagicMock
from unittest.mock import AsyncMock, MagicMock, patch
from fastapi.testclient import TestClient
from litellm.proxy.proxy_server import app
# Mock the global handler instance in token_counter module
@ -678,57 +614,6 @@ async def test_factory_registration():
assert not counter.should_use_token_counting_api(custom_llm_provider=None)
@pytest.mark.skip(
reason="Requires Google/Vertex AI credentials (GEMINI_API_KEY or VERTEX_AI_PRIVATE_KEY)."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", ["gemini-2.5-pro", "vertex-ai-gemini-2.5-pro"])
async def test_vertex_ai_gemini_token_counting_with_contents(model_name):
"""
Test token counting for Vertex AI Gemini model using contents format with call_endpoint=True
"""
load_vertex_ai_credentials()
llm_router = Router(
model_list=[
{
"model_name": "gemini-2.5-pro",
"litellm_params": {
"model": "gemini/gemini-2.5-pro",
},
},
{
"model_name": "vertex-ai-gemini-2.5-pro",
"litellm_params": {
"model": "vertex_ai/gemini-2.5-pro",
},
},
]
)
setattr(litellm.proxy.proxy_server, "llm_router", llm_router)
# Test with contents format and call_endpoint=True
response = await token_counter(
request=TokenCountRequest(
model=model_name,
contents=[
{"parts": [{"text": "Hello world, how are you doing today? i am ij"}]}
],
),
call_endpoint=True,
)
print("Vertex AI Gemini token counting response:", response)
# validate we have original response
assert response.original_response is not None
assert response.original_response.get("totalTokens") is not None
assert response.original_response.get("promptTokensDetails") is not None
prompt_tokens_details = response.original_response.get("promptTokensDetails")
assert prompt_tokens_details is not None
@pytest.mark.asyncio
async def test_bedrock_count_tokens_endpoint():
"""
@ -779,7 +664,7 @@ async def test_vertex_ai_anthropic_token_counting():
This tests the token counting implementation for Vertex AI partner models
without making actual API calls. Mocks at the handler level to test the full flow.
"""
from unittest.mock import AsyncMock, patch, MagicMock
from unittest.mock import patch
# Mock the Vertex AI partner models token counter response
mock_token_response = {

View file

@ -438,7 +438,7 @@ def test_is_request_body_safe_global_enabled(
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "gpt-3.5-turbo",
"api_key": os.getenv("OPENAI_API_KEY"),
"api_key": "sk-openai-unit-test",
},
}
]
@ -475,7 +475,7 @@ def test_is_request_body_safe_model_enabled(
"model_name": "fireworks_ai/*",
"litellm_params": {
"model": "fireworks_ai/*",
"api_key": os.getenv("FIREWORKS_API_KEY"),
"api_key": "sk-fireworks-unit-test",
"configurable_clientside_auth_params": (
["api_base"] if allow_client_side_credentials else []
),