mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
ci: keep project secrets out of litellm-tests and move key-dependent tests to their tiers
This commit is contained in:
parent
8376a5a431
commit
e68cf49445
15 changed files with 232 additions and 327 deletions
|
|
@ -50,7 +50,7 @@ legacy_paths() {
|
|||
mcp-integration)
|
||||
echo tests/unit/proxy/_experimental/mcp_server
|
||||
echo tests/unit/responses/mcp
|
||||
echo tests/integration/mcp_sdk1 ;;
|
||||
echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
|
||||
proxy-db-auth-checks)
|
||||
echo tests/unit/proxy/auth/test_auth_checks.py
|
||||
echo tests/unit/proxy/auth/test_user_api_key_auth.py
|
||||
|
|
@ -105,7 +105,6 @@ legacy_paths() {
|
|||
echo tests/unit/proxy/auth/test_proxy_routes.py
|
||||
echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
|
||||
echo tests/unit/proxy/test_proxy_config_unit_test.py
|
||||
echo tests/unit/proxy/test_proxy_pass_user_config.py
|
||||
echo tests/unit/proxy/test_proxy_token_counter.py
|
||||
echo tests/unit/proxy/test_server_root_path.py ;;
|
||||
proxy-db-proxy-server-core)
|
||||
|
|
|
|||
|
|
@ -186,6 +186,9 @@ jobs:
|
|||
legacy_mcp_peer:
|
||||
type: boolean
|
||||
default: false
|
||||
reruns:
|
||||
type: integer
|
||||
default: 0
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
|
|
@ -214,14 +217,19 @@ jobs:
|
|||
no_output_timeout: 20m
|
||||
command: |
|
||||
mkdir -p test-results/<< parameters.flag >>
|
||||
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)"
|
||||
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
|
||||
[ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
|
||||
mapfile -t files < <(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)
|
||||
if [ "${#files[@]}" -eq 0 ]; then echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; fi
|
||||
xdist_args=()
|
||||
if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
|
||||
rerun_args=(-p no:rerunfailures)
|
||||
if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
|
||||
test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
|
||||
set +e
|
||||
uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
|
||||
env -i "${test_env[@]}" \
|
||||
uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
|
||||
|
|
@ -282,7 +290,7 @@ jobs:
|
|||
- start_redis
|
||||
- run:
|
||||
name: Run owned integration contracts
|
||||
command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
|
||||
command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Stop owned database and Redis
|
||||
|
|
@ -309,6 +317,7 @@ workflows:
|
|||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
workers: 2
|
||||
reruns: 2
|
||||
matrix:
|
||||
parameters:
|
||||
flag: [caching-local, proxy-extras, enterprise-routing]
|
||||
|
|
@ -325,6 +334,7 @@ workflows:
|
|||
- unit:
|
||||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
reruns: 2
|
||||
matrix:
|
||||
parameters:
|
||||
flag:
|
||||
|
|
@ -346,6 +356,7 @@ workflows:
|
|||
name: unit-proxy-db-proxy-utils
|
||||
flag: proxy-db-proxy-utils
|
||||
shards: 1
|
||||
reruns: 2
|
||||
dist: worksteal
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
|
|
@ -354,6 +365,7 @@ workflows:
|
|||
flag: proxy-db-key-generation
|
||||
shards: 1
|
||||
workers: 0
|
||||
reruns: 2
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- documentation
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
"""Live e2e: `/utils/token_counter?call_endpoint=true` counts Gemini `contents` upstream.
|
||||
|
||||
Google's countTokens API is the only tokenizer that knows Gemini's real token
|
||||
boundaries, so the proxy must forward `contents` to it for both the AI Studio and
|
||||
Vertex deployments and hand back the provider's `promptTokensDetails`. Claude on
|
||||
Vertex is covered by `/v1/messages/count_tokens`; this is the Gemini `contents`
|
||||
route the claude_code rows never reach
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
GEMINI_DEPLOYMENTS = ("gemini-2.5-flash", "gemini-2.5-flash-vertex")
|
||||
|
||||
|
||||
class _Part(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class _Content(BaseModel):
|
||||
parts: tuple[_Part, ...]
|
||||
|
||||
|
||||
class _TokenCountBody(BaseModel):
|
||||
model: str
|
||||
contents: tuple[_Content, ...]
|
||||
|
||||
|
||||
class _CallEndpoint(BaseModel):
|
||||
call_endpoint: bool = True
|
||||
|
||||
|
||||
class _ModalityTokens(BaseModel):
|
||||
modality: str
|
||||
tokenCount: int
|
||||
|
||||
|
||||
class _CountTokensUpstream(BaseModel):
|
||||
totalTokens: int
|
||||
promptTokensDetails: tuple[_ModalityTokens, ...]
|
||||
|
||||
|
||||
class _TokenCountResponse(BaseModel):
|
||||
total_tokens: int
|
||||
request_model: str
|
||||
model_used: str
|
||||
tokenizer_type: str
|
||||
original_response: _CountTokensUpstream
|
||||
|
||||
|
||||
class TestGeminiContentsTokenCounting:
|
||||
@pytest.mark.parametrize("model", GEMINI_DEPLOYMENTS)
|
||||
def test_contents_are_counted_by_the_provider_endpoint(
|
||||
self, proxy: ProxyClient, scoped_key: str, model: str
|
||||
) -> None:
|
||||
text = f"Hello world, how are you doing today? {unique_marker()}"
|
||||
body = _TokenCountBody(model=model, contents=(_Content(parts=(_Part(text=text),)),))
|
||||
|
||||
result = proxy.transport.send(
|
||||
"/utils/token_counter",
|
||||
headers=proxy.transport.bearer(scoped_key),
|
||||
json=body,
|
||||
params=_CallEndpoint(),
|
||||
)
|
||||
|
||||
require_successful_call(result)
|
||||
counted = _TokenCountResponse.model_validate_json(result.body)
|
||||
assert counted.request_model == model, counted
|
||||
assert counted.original_response.totalTokens == counted.total_tokens > 0, counted
|
||||
assert counted.original_response.promptTokensDetails, counted
|
||||
assert all(detail.tokenCount > 0 for detail in counted.original_response.promptTokensDetails), counted
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
import asyncio
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def event_loop():
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
yield loop
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
|
||||
"""
|
||||
importlib.reload(litellm)
|
||||
import asyncio
|
||||
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
yield
|
||||
|
||||
# Teardown code (executes after the yield point)
|
||||
# LoggingWorker carries still-queued coroutines onto the next test's loop, where they'd log into that test's callbacks
|
||||
asyncio.run(GLOBAL_LOGGING_WORKER.clear_queue())
|
||||
loop.close() # Close the loop created earlier
|
||||
asyncio.set_event_loop(None) # Remove the reference to the loop
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
async def drain_logging_worker():
|
||||
"""
|
||||
The logging queue is bound to the running loop, so anything left queued when a test's loop
|
||||
goes away is carried onto the next test's loop and fires against its callbacks.
|
||||
"""
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
yield
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.clear_queue(), timeout=10)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
103
tests/integration/routing/test_user_config_routing.py
Normal file
103
tests/integration/routing/test_user_config_routing.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
USER_KEY: Final = "sk-user-supplied-" + uuid.uuid4().hex
|
||||
|
||||
|
||||
def _completion(request: Request) -> Reply:
|
||||
if request.target != "/v1/chat/completions":
|
||||
return Reply(status=404, body=b"{}")
|
||||
body: Final = json.loads(request.body)
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-user-config",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": body["model"],
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _user_config(upstream_url: str) -> dict[str, object]:
|
||||
return {
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "user-config-deployment",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4.1-mini",
|
||||
"api_base": upstream_url + "/v1",
|
||||
"api_key": USER_KEY,
|
||||
},
|
||||
}
|
||||
],
|
||||
"num_retries": 0,
|
||||
}
|
||||
|
||||
|
||||
def _opt_in_config(directory: Path, upstream_url: str) -> Path:
|
||||
config: Final = directory / "allow_client_side_credentials_config.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "admin-deployment",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4.1-mini",
|
||||
"api_base": upstream_url + "/v1",
|
||||
"api_key": "sk-admin-configured",
|
||||
},
|
||||
}
|
||||
],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"store_model_in_db": True,
|
||||
"allow_client_side_credentials": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _request_body(upstream_url: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "user-config-deployment",
|
||||
"messages": [{"role": "user", "content": "user config control"}],
|
||||
"user_config": _user_config(upstream_url),
|
||||
}
|
||||
|
||||
|
||||
def test_user_config_routes_to_the_user_supplied_deployment_when_opted_in(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with wire_server(_completion) as upstream:
|
||||
config: Final = _opt_in_config(tmp_path, upstream.url)
|
||||
with owned_proxy(gateway, tmp_path, {}, config=config) as candidate:
|
||||
response: Final = candidate.request("POST", "/v1/chat/completions", _request_body(upstream.url))
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["choices"][0]["message"]["content"] == "routed"
|
||||
outbound: Final = tuple(upstream.received.get_nowait() for _ in range(upstream.received.qsize()))
|
||||
completions: Final = tuple(request for request in outbound if request.target == "/v1/chat/completions")
|
||||
assert len(completions) == 1, outbound
|
||||
assert completions[0].headers["authorization"] == f"Bearer {USER_KEY}"
|
||||
assert json.loads(completions[0].body)["model"] == "gpt-4.1-mini"
|
||||
|
||||
|
||||
def test_user_config_is_rejected_without_the_opt_in(gateway: Gateway) -> None:
|
||||
with wire_server(_completion) as upstream:
|
||||
response: Final = gateway.request("POST", "/v1/chat/completions", _request_body(upstream.url))
|
||||
assert response.status_code == 401, response.text
|
||||
assert "user_config is not allowed in request body" in response.text
|
||||
assert upstream.received.empty()
|
||||
|
|
@ -17,7 +17,7 @@ mcp_servers:
|
|||
transport: stdio
|
||||
command: python3
|
||||
args:
|
||||
- tests/integration/mcp_sdk1/mcp_math_server.py
|
||||
- tests/mcp_tests/mcp_server.py
|
||||
allow_all_keys: true # Needed for master_key auth when DB/object_permission not used
|
||||
math_streamable_http:
|
||||
transport: http
|
||||
|
|
@ -18,8 +18,8 @@ import json
|
|||
async def test_mcp_agent():
|
||||
"""Test MCP agent functionality with a simple math server"""
|
||||
try:
|
||||
local_server_path = "./mcp_math_server.py"
|
||||
ci_cd_server_path = "tests/integration/mcp_sdk1/mcp_math_server.py"
|
||||
local_server_path = "./mcp_server.py"
|
||||
ci_cd_server_path = "tests/mcp_tests/mcp_server.py"
|
||||
|
||||
# Use the correct path for the server
|
||||
server_path = (
|
||||
|
|
|
|||
|
|
@ -35,10 +35,10 @@ from litellm.proxy.proxy_server import (
|
|||
initialize,
|
||||
)
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
CONFIG_TEMPLATE_PATH = PROJECT_ROOT / "tests/integration/mcp_sdk1/test_config_mcp_e2e.yaml"
|
||||
MCP_SERVER_SCRIPT = PROJECT_ROOT / "tests/integration/mcp_sdk1/mcp_math_server.py"
|
||||
CONFIG_TEMPLATE_PATH = Path("tests/mcp_tests/test_configs/test_config_mcp_e2e.yaml")
|
||||
MCP_SERVER_SCRIPT = Path("tests/mcp_tests/mcp_server.py")
|
||||
MCP_PEER_PYTHON = os.environ.get("MCP_TEST_PEER_PYTHON", sys.executable)
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
PROXY_START_TIMEOUT = 30
|
||||
|
||||
|
||||
|
|
@ -13,7 +13,6 @@ import asyncio
|
|||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
import os
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -165,9 +164,9 @@ async def test_prometheus_metric_tracking():
|
|||
"model_name": "gpt-5-mini", # openai model name
|
||||
"litellm_params": { # params for litellm completion/embedding call
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": os.getenv("AZURE_AI_API_KEY"),
|
||||
"api_version": os.getenv("AZURE_AI_API_VERSION"),
|
||||
"api_base": os.getenv("AZURE_AI_API_BASE"),
|
||||
"api_key": "sk-azure-unit-test",
|
||||
"api_version": "2025-01-01-preview",
|
||||
"api_base": "https://unit-test.openai.azure.com",
|
||||
},
|
||||
"model_info": {"id": "azure-model-id"},
|
||||
},
|
||||
|
|
@ -180,9 +179,6 @@ async def test_prometheus_metric_tracking():
|
|||
},
|
||||
],
|
||||
provider_budget_config=provider_budget_config,
|
||||
redis_host=os.getenv("REDIS_HOST"),
|
||||
redis_port=int(os.getenv("REDIS_PORT", 6379)),
|
||||
redis_password=os.getenv("REDIS_PASSWORD"),
|
||||
)
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -53,8 +53,7 @@ def test_custom_auth(client):
|
|||
"max_tokens": 10,
|
||||
}
|
||||
# Your bearer token
|
||||
token = os.getenv("PROXY_MASTER_KEY")
|
||||
print(f"token: {token}")
|
||||
token = "sk-unit-test-master"
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
with pytest.raises(Exception, match="Authentication Error, Failed custom auth") as exc_info:
|
||||
client.post("/chat/completions", json=test_data, headers=headers)
|
||||
|
|
@ -71,7 +70,7 @@ def test_custom_auth_bearer(client):
|
|||
"max_tokens": 10,
|
||||
}
|
||||
# Your bearer token
|
||||
token = os.getenv("PROXY_MASTER_KEY")
|
||||
token = "sk-unit-test-master"
|
||||
|
||||
headers = {"Authorization": f"WITHOUT BEAR Er {token}"}
|
||||
with pytest.raises(Exception, match="CustomAuth - Malformed API Key passed in") as exc_info:
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
import sys, os
|
||||
import traceback
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
import io
|
||||
|
||||
# this file is to test litellm/proxy
|
||||
|
||||
import pytest, logging, asyncio
|
||||
import litellm
|
||||
from litellm import embedding, completion, completion_cost, Timeout
|
||||
from litellm import RateLimitError
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG, # Set the desired logging level
|
||||
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
|
||||
# test /chat/completion request to the proxy
|
||||
from fastapi.testclient import TestClient
|
||||
from fastapi import FastAPI
|
||||
from litellm.proxy.proxy_server import (
|
||||
router,
|
||||
save_worker_config,
|
||||
initialize,
|
||||
) # Replace with the actual module where your FastAPI router is defined
|
||||
|
||||
# Your bearer token
|
||||
token = "sk-1234"
|
||||
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def client_no_auth():
|
||||
# Assuming litellm.proxy.proxy_server is an object
|
||||
from litellm.proxy.proxy_server import cleanup_router_config_variables
|
||||
|
||||
cleanup_router_config_variables()
|
||||
filepath = os.path.dirname(os.path.abspath(__file__))
|
||||
config_fp = f"{filepath}/test_configs/test_config_no_auth.yaml"
|
||||
# initialize can get run in parallel, it sets specific variables for the fast api app, sinc eit gets run in parallel different tests use the wrong variables
|
||||
asyncio.run(initialize(config=config_fp, debug=True))
|
||||
app = FastAPI()
|
||||
app.include_router(router) # Include your router in the test app
|
||||
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("AZURE_AI_API_KEY") is None
|
||||
or os.environ.get("OPENAI_API_KEY") is None,
|
||||
reason="AZURE_AI_API_KEY or OPENAI_API_KEY not set - skipping integration test",
|
||||
)
|
||||
def test_chat_completion(client_no_auth):
|
||||
global headers
|
||||
|
||||
from litellm.types.router import RouterConfig, ModelConfig
|
||||
from litellm.types.completion import CompletionRequest
|
||||
|
||||
user_config = RouterConfig(
|
||||
model_list=[
|
||||
ModelConfig(
|
||||
model_name="user-azure-instance",
|
||||
litellm_params=CompletionRequest(
|
||||
model="azure/gpt-4.1-mini",
|
||||
api_key=os.getenv("AZURE_AI_API_KEY"),
|
||||
api_version=os.getenv("AZURE_API_VERSION"),
|
||||
api_base=os.getenv("AZURE_AI_API_BASE"),
|
||||
timeout=10,
|
||||
),
|
||||
tpm=240000,
|
||||
rpm=1800,
|
||||
),
|
||||
ModelConfig(
|
||||
model_name="user-openai-instance",
|
||||
litellm_params=CompletionRequest(
|
||||
model="gpt-3.5-turbo",
|
||||
api_key=os.getenv("OPENAI_API_KEY"),
|
||||
timeout=10,
|
||||
),
|
||||
tpm=240000,
|
||||
rpm=1800,
|
||||
),
|
||||
],
|
||||
num_retries=2,
|
||||
allowed_fails=3,
|
||||
fallbacks=[{"user-azure-instance": ["user-openai-instance"]}],
|
||||
).dict()
|
||||
|
||||
try:
|
||||
# Your test data
|
||||
test_data = {
|
||||
"model": "user-azure-instance",
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
],
|
||||
"max_tokens": 10,
|
||||
"user_config": user_config,
|
||||
}
|
||||
|
||||
print("testing proxy server with chat completions")
|
||||
response = client_no_auth.post("/v1/chat/completions", json=test_data)
|
||||
print(f"response - {response.text}")
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
print(f"Received response: {result}")
|
||||
except Exception as e:
|
||||
pytest.fail(f"LiteLLM Proxy test failed. Exception - {str(e)}")
|
||||
|
||||
|
||||
# Run the test
|
||||
|
|
@ -2,10 +2,7 @@
|
|||
# 1. Generate a Key, and use it to make a call
|
||||
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -35,79 +32,6 @@ from litellm.types.utils import TokenCountResponse
|
|||
verbose_proxy_logger.setLevel(level=logging.DEBUG)
|
||||
|
||||
|
||||
def get_vertex_ai_creds_json() -> dict:
|
||||
# Define the path to the vertex_key.json file
|
||||
print("loading vertex ai credentials")
|
||||
filepath = os.path.dirname(os.path.abspath(__file__))
|
||||
vertex_key_path = filepath + "/vertex_key.json"
|
||||
# Read the existing content of the file or create an empty dictionary
|
||||
try:
|
||||
with open(vertex_key_path, "r") as file:
|
||||
# Read the file content
|
||||
print("Read vertexai file path")
|
||||
content = file.read()
|
||||
|
||||
# If the file is empty or not valid JSON, create an empty dictionary
|
||||
if not content or not content.strip():
|
||||
service_account_key_data = {}
|
||||
else:
|
||||
# Attempt to load the existing JSON content
|
||||
file.seek(0)
|
||||
service_account_key_data = json.load(file)
|
||||
except FileNotFoundError:
|
||||
# If the file doesn't exist, create an empty dictionary
|
||||
service_account_key_data = {}
|
||||
|
||||
# Update the service_account_key_data with environment variables
|
||||
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
|
||||
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
|
||||
private_key = private_key.replace("\\n", "\n")
|
||||
service_account_key_data["private_key_id"] = private_key_id
|
||||
service_account_key_data["private_key"] = private_key
|
||||
|
||||
return service_account_key_data
|
||||
|
||||
|
||||
def load_vertex_ai_credentials():
|
||||
# Define the path to the vertex_key.json file
|
||||
print("loading vertex ai credentials")
|
||||
filepath = os.path.dirname(os.path.abspath(__file__))
|
||||
vertex_key_path = filepath + "/vertex_key.json"
|
||||
|
||||
# Read the existing content of the file or create an empty dictionary
|
||||
try:
|
||||
with open(vertex_key_path, "r") as file:
|
||||
# Read the file content
|
||||
print("Read vertexai file path")
|
||||
content = file.read()
|
||||
|
||||
# If the file is empty or not valid JSON, create an empty dictionary
|
||||
if not content or not content.strip():
|
||||
service_account_key_data = {}
|
||||
else:
|
||||
# Attempt to load the existing JSON content
|
||||
file.seek(0)
|
||||
service_account_key_data = json.load(file)
|
||||
except FileNotFoundError:
|
||||
# If the file doesn't exist, create an empty dictionary
|
||||
service_account_key_data = {}
|
||||
|
||||
# Update the service_account_key_data with environment variables
|
||||
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
|
||||
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
|
||||
private_key = private_key.replace("\\n", "\n")
|
||||
service_account_key_data["private_key_id"] = private_key_id
|
||||
service_account_key_data["private_key"] = private_key
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
|
||||
# Write the updated content to the temporary files
|
||||
json.dump(service_account_key_data, temp_file, indent=2)
|
||||
|
||||
# Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vLLM_token_counting():
|
||||
"""
|
||||
|
|
@ -223,10 +147,12 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
- Should return response in Anthropic format: {"input_tokens": <count>}
|
||||
- Should work as wrapper around internal token_counter function
|
||||
"""
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
from fastapi import Request
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
|
||||
# Mock request object
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request_data = {
|
||||
|
|
@ -295,10 +221,12 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
- Should still work and return Anthropic format
|
||||
- Should call internal token_counter with from_anthropic_endpoint=True
|
||||
"""
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
from fastapi import Request
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
|
||||
# Mock request object
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request_data = {
|
||||
|
|
@ -435,10 +363,12 @@ async def test_anthropic_endpoint_error_handling():
|
|||
"""
|
||||
Test error handling in the /v1/messages/count_tokens endpoint
|
||||
"""
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
from fastapi import Request, HTTPException
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import count_tokens
|
||||
|
||||
# Mock request object
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_user_api_key_dict = MagicMock()
|
||||
|
|
@ -474,8 +404,10 @@ async def test_anthropic_endpoint_error_handling():
|
|||
@pytest.mark.asyncio
|
||||
async def test_factory_anthropic_endpoint_calls_anthropic_counter():
|
||||
"""Test that /v1/messages/count_tokens with Anthropic model uses Anthropic counter."""
|
||||
from unittest.mock import patch, AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
# Mock the global handler instance in token_counter module
|
||||
|
|
@ -531,8 +463,10 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter():
|
|||
@pytest.mark.asyncio
|
||||
async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
|
||||
"""Test that /v1/messages/count_tokens with GPT-4 does NOT use Anthropic counter."""
|
||||
from unittest.mock import patch, AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
# Mock the global handler instance in token_counter module
|
||||
|
|
@ -590,8 +524,10 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
|
|||
@pytest.mark.asyncio
|
||||
async def test_factory_normal_token_counter_endpoint_does_not_call_anthropic():
|
||||
"""Test that /utils/token_counter does NOT use Anthropic counter even with Anthropic model."""
|
||||
from unittest.mock import patch, AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
# Mock the global handler instance in token_counter module
|
||||
|
|
@ -678,57 +614,6 @@ async def test_factory_registration():
|
|||
assert not counter.should_use_token_counting_api(custom_llm_provider=None)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Requires Google/Vertex AI credentials (GEMINI_API_KEY or VERTEX_AI_PRIVATE_KEY)."
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model_name", ["gemini-2.5-pro", "vertex-ai-gemini-2.5-pro"])
|
||||
async def test_vertex_ai_gemini_token_counting_with_contents(model_name):
|
||||
"""
|
||||
Test token counting for Vertex AI Gemini model using contents format with call_endpoint=True
|
||||
"""
|
||||
load_vertex_ai_credentials()
|
||||
llm_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gemini-2.5-pro",
|
||||
"litellm_params": {
|
||||
"model": "gemini/gemini-2.5-pro",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "vertex-ai-gemini-2.5-pro",
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/gemini-2.5-pro",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "llm_router", llm_router)
|
||||
|
||||
# Test with contents format and call_endpoint=True
|
||||
response = await token_counter(
|
||||
request=TokenCountRequest(
|
||||
model=model_name,
|
||||
contents=[
|
||||
{"parts": [{"text": "Hello world, how are you doing today? i am ij"}]}
|
||||
],
|
||||
),
|
||||
call_endpoint=True,
|
||||
)
|
||||
|
||||
print("Vertex AI Gemini token counting response:", response)
|
||||
|
||||
# validate we have original response
|
||||
assert response.original_response is not None
|
||||
assert response.original_response.get("totalTokens") is not None
|
||||
assert response.original_response.get("promptTokensDetails") is not None
|
||||
|
||||
prompt_tokens_details = response.original_response.get("promptTokensDetails")
|
||||
assert prompt_tokens_details is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_count_tokens_endpoint():
|
||||
"""
|
||||
|
|
@ -779,7 +664,7 @@ async def test_vertex_ai_anthropic_token_counting():
|
|||
This tests the token counting implementation for Vertex AI partner models
|
||||
without making actual API calls. Mocks at the handler level to test the full flow.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
from unittest.mock import patch
|
||||
|
||||
# Mock the Vertex AI partner models token counter response
|
||||
mock_token_response = {
|
||||
|
|
|
|||
|
|
@ -438,7 +438,7 @@ def test_is_request_body_safe_global_enabled(
|
|||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"api_key": "sk-openai-unit-test",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
|
@ -475,7 +475,7 @@ def test_is_request_body_safe_model_enabled(
|
|||
"model_name": "fireworks_ai/*",
|
||||
"litellm_params": {
|
||||
"model": "fireworks_ai/*",
|
||||
"api_key": os.getenv("FIREWORKS_API_KEY"),
|
||||
"api_key": "sk-fireworks-unit-test",
|
||||
"configurable_clientside_auth_params": (
|
||||
["api_base"] if allow_client_side_credentials else []
|
||||
),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue