mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 177e2b56a2 into 3b8ad37296
This commit is contained in:
commit
d43708fe99
6 changed files with 306 additions and 25 deletions
|
|
@ -8,7 +8,7 @@ from typing import Final
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -48,7 +48,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
openai_error_type,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields, counts_with_billed_request
|
||||
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
|
||||
from litellm.router import Router
|
||||
|
||||
|
|
@ -169,6 +169,20 @@ async def anthropic_response(
|
|||
|
||||
This was a BETA endpoint that calls 100+ LLMs in the anthropic format.
|
||||
"""
|
||||
return await _anthropic_messages_response(
|
||||
ProxyBaseLLMRequestProcessing(data=await read_request_body(request=request)),
|
||||
fastapi_response,
|
||||
request,
|
||||
user_api_key_dict,
|
||||
)
|
||||
|
||||
|
||||
async def _anthropic_messages_response(
|
||||
base_llm_response_processor: ProxyBaseLLMRequestProcessing,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> object:
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
|
|
@ -182,8 +196,7 @@ async def anthropic_response(
|
|||
version,
|
||||
)
|
||||
|
||||
data: Final = await read_request_body(request=request)
|
||||
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
data: Final = base_llm_response_processor.data
|
||||
try:
|
||||
result: Final = await base_llm_response_processor.base_process_llm_request(
|
||||
request=request,
|
||||
|
|
@ -244,7 +257,7 @@ async def anthropic_response(
|
|||
usage=_usage,
|
||||
)
|
||||
|
||||
if data.get("stream", None) is not None and data["stream"] is True:
|
||||
if "stream" in data and data["stream"] is True:
|
||||
# For streaming, use the standard SSE data generator
|
||||
async def _passthrough_stream_generator():
|
||||
yield _anthropic_response
|
||||
|
|
@ -334,6 +347,43 @@ def _requested_output_format(data: Mapping[str, JsonValue]) -> JsonValue:
|
|||
_COUNTED_LITELLM_PARAMS: Final = ("additional_drop_params", "drop_params")
|
||||
|
||||
|
||||
class _BilledUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
input_tokens: int
|
||||
cache_read_input_tokens: int | None = None
|
||||
cache_creation_input_tokens: int | None = None
|
||||
|
||||
@property
|
||||
def billed_input_tokens(self) -> int:
|
||||
return self.input_tokens + (self.cache_read_input_tokens or 0) + (self.cache_creation_input_tokens or 0)
|
||||
|
||||
|
||||
class _BilledResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
usage: _BilledUsage
|
||||
|
||||
|
||||
def _counts_with_billed_request(llm_router: Router | None, model: str, team_id: str | None) -> bool:
|
||||
if llm_router is None:
|
||||
return False
|
||||
return counts_with_billed_request(llm_router.get_model_list(model_name=model, team_id=team_id) or ())
|
||||
|
||||
|
||||
async def _billed_input_tokens(
|
||||
probe: ProxyBaseLLMRequestProcessing,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> object:
|
||||
response: Final = await _anthropic_messages_response(probe, fastapi_response, request, user_api_key_dict)
|
||||
if isinstance(response, Response):
|
||||
return response
|
||||
billed: Final = _BilledResponse.model_validate(response, from_attributes=True)
|
||||
return {"input_tokens": billed.usage.billed_input_tokens, "litellm_count_source": "billed_request"}
|
||||
|
||||
|
||||
class _CountTokensBodyOptions(TypedDict):
|
||||
output_format: ReadOnly[JsonValue]
|
||||
context_management: ReadOnly[JsonValue]
|
||||
|
|
@ -357,6 +407,7 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params:
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def count_tokens(
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # Used for auth
|
||||
):
|
||||
|
|
@ -397,6 +448,18 @@ async def count_tokens(
|
|||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="messages parameter is required")
|
||||
|
||||
if isinstance(model_name, str) and _counts_with_billed_request(
|
||||
llm_router, model_name, user_api_key_dict.team_id
|
||||
):
|
||||
return await _billed_input_tokens(
|
||||
ProxyBaseLLMRequestProcessing(
|
||||
data={**request_data, "max_tokens": 1, "stream": False, "disable_fallbacks": True}
|
||||
),
|
||||
fastapi_response,
|
||||
request,
|
||||
user_api_key_dict,
|
||||
)
|
||||
|
||||
auto_drop_params: Final = should_auto_drop_params_for_agentic_cli(
|
||||
request.headers.get("user-agent", ""), "drop_params" in data, proxy_config
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
|
@ -176,6 +176,13 @@ def resolve_token_count_settings(
|
|||
)
|
||||
|
||||
|
||||
def counts_with_billed_request(deployments: Sequence[Mapping[str, object]]) -> bool:
|
||||
return bool(deployments) and all(
|
||||
(as_str_mapping(deployment.get("model_info")) or {}).get("token_count_billed_request") is True
|
||||
for deployment in deployments
|
||||
)
|
||||
|
||||
|
||||
def count_source_fields(response: TokenCountResponse) -> TokenCountSourceFields:
|
||||
match response.count_source:
|
||||
case None:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from typing import Literal, TypeAlias
|
|||
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate"]
|
||||
TokenCountSource: TypeAlias = Literal["provider", "token_count_model", "local_estimate", "billed_request"]
|
||||
TokenCountTokenizer: TypeAlias = Literal["kimi_k3"]
|
||||
TokenCountProvider: TypeAlias = Literal["bedrock", "bedrock_mantle"]
|
||||
|
||||
|
|
|
|||
|
|
@ -5,12 +5,18 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec
|
|||
import json
|
||||
import logging
|
||||
import unittest
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import Request, Response
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.types import Message
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
|
|
@ -462,7 +468,7 @@ class TestErrorLogCarriesCallId:
|
|||
), # test-quality-ok: module global imported at call time; the test targets the endpoint's except block
|
||||
caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"),
|
||||
):
|
||||
response = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response.status_code == 500
|
||||
record = self._error_record(caplog)
|
||||
|
|
@ -514,7 +520,7 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod
|
|||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts it is never reached
|
||||
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body)["error"]["type"] == "invalid_request_error"
|
||||
|
|
@ -537,7 +543,7 @@ async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router
|
|||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
patch.object(proxy_server, "llm_router", new=_claude_router()), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
assert counter.await_args.kwargs["request"].tools == _MALFORMED_TOOL_ENTRY
|
||||
|
|
@ -570,7 +576,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
|
|||
patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam
|
||||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
counted: Final = counter.await_args.kwargs["request"]
|
||||
|
|
@ -609,12 +615,213 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
|
|||
patch.object(proxy_server, "count_request_tokens", new=counter), # test-quality-ok: module global imported at call time; the test asserts what it receives
|
||||
patch.object(proxy_server, "llm_router", new=router), # test-quality-ok: module global imported at call time; no injection seam on this route
|
||||
):
|
||||
response: Final = await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
response: Final = await ep.count_tokens(fastapi_response=Response(), request=request, user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 7}
|
||||
assert counter.await_args.kwargs["request"].tools is None
|
||||
|
||||
|
||||
_ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages"
|
||||
_ANTHROPIC_COUNT_TOKENS_URL: Final = "https://api.anthropic.com/v1/messages/count_tokens"
|
||||
_OPTED_IN: Final = {"token_count_billed_request": True}
|
||||
_WEATHER_TOOL: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather for a city",
|
||||
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
}
|
||||
_COUNT_BODY: Final = {
|
||||
"model": "claude-count",
|
||||
"max_tokens": 64,
|
||||
"system": [{"type": "text", "text": "Answer in one word."}],
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}],
|
||||
}
|
||||
|
||||
|
||||
def _count_router(*model_infos: Mapping[str, object]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-count",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
|
||||
"model_info": dict(model_info),
|
||||
}
|
||||
for model_info in model_infos
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
|
||||
def _count_request(body: Mapping[str, object]) -> Request:
|
||||
payload: Final = json.dumps(body).encode()
|
||||
|
||||
async def receive() -> Message:
|
||||
return {"type": "http.request", "body": payload, "more_body": False}
|
||||
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages/count_tokens",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
|
||||
|
||||
def _billed_reply(model: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_count",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": "Sunny"}],
|
||||
"stop_reason": "max_tokens",
|
||||
"usage": {
|
||||
"input_tokens": 1200,
|
||||
"cache_read_input_tokens": 30,
|
||||
"cache_creation_input_tokens": 4,
|
||||
"output_tokens": 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def count_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("count_proxy")
|
||||
async def test_count_tokens_answers_the_billed_input_of_a_one_token_request_when_every_deployment_opts_in(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN, _OPTED_IN))
|
||||
fastapi_response: Final = Response()
|
||||
|
||||
with respx.mock:
|
||||
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
|
||||
count_route: Final = respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 9})
|
||||
)
|
||||
response: Final = await ep.count_tokens(
|
||||
fastapi_response, _count_request({**_COUNT_BODY, "stream": True}), UserAPIKeyAuth()
|
||||
)
|
||||
|
||||
assert response == {"input_tokens": 1234, "litellm_count_source": "billed_request"}
|
||||
assert not count_route.called
|
||||
sent: Final = json.loads(messages_route.calls.last.request.content)
|
||||
assert sent["max_tokens"] == 1
|
||||
assert sent.get("stream", False) is False
|
||||
assert sent["system"] == _COUNT_BODY["system"]
|
||||
assert sent["tools"] == _COUNT_BODY["tools"]
|
||||
assert sent["messages"] == _COUNT_BODY["messages"]
|
||||
assert float(fastapi_response.headers["x-litellm-response-cost"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("count_proxy")
|
||||
@pytest.mark.parametrize(
|
||||
"model_infos",
|
||||
[({}, {}), (_OPTED_IN, {}), ({"token_count_billed_request": "true"},)],
|
||||
ids=["knob_off", "one_of_two_deployments", "string_value"],
|
||||
)
|
||||
async def test_count_tokens_keeps_the_provider_count_unless_every_deployment_opts_in(
|
||||
monkeypatch: pytest.MonkeyPatch, model_infos: tuple[Mapping[str, object], ...]
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _count_router(*model_infos))
|
||||
|
||||
with respx.mock:
|
||||
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
|
||||
respx.post(_ANTHROPIC_COUNT_TOKENS_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9}))
|
||||
response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth())
|
||||
|
||||
assert response == {"input_tokens": 9, "litellm_count_source": "provider"}
|
||||
assert not messages_route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("count_proxy")
|
||||
async def test_count_tokens_estimates_a_model_with_no_deployments_without_a_billed_request(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _count_router(_OPTED_IN))
|
||||
|
||||
with respx.mock:
|
||||
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(return_value=_billed_reply("claude-sonnet-5-5"))
|
||||
response: Final = await ep.count_tokens(
|
||||
Response(), _count_request({**_COUNT_BODY, "model": "claude-unlisted"}), UserAPIKeyAuth()
|
||||
)
|
||||
|
||||
assert response["litellm_count_source"] == "local_estimate"
|
||||
assert not messages_route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("count_proxy")
|
||||
async def test_count_tokens_answers_a_failed_billed_request_as_its_error_without_falling_back(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-count",
|
||||
"litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_key": "sk-test"},
|
||||
"model_info": dict(_OPTED_IN),
|
||||
},
|
||||
{
|
||||
"model_name": "claude-backup",
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "sk-test"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"claude-count": ["claude-backup"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
def reply(request: httpx.Request) -> httpx.Response:
|
||||
model: Final = json.loads(request.content)["model"]
|
||||
if model == "claude-haiku-5-5":
|
||||
return _billed_reply(model)
|
||||
return httpx.Response(
|
||||
429, json={"type": "error", "error": {"type": "rate_limit_error", "message": "Number of requests is high"}}
|
||||
)
|
||||
|
||||
with respx.mock:
|
||||
messages_route: Final = respx.post(_ANTHROPIC_MESSAGES_URL).mock(side_effect=reply)
|
||||
response: Final = await ep.count_tokens(Response(), _count_request(_COUNT_BODY), UserAPIKeyAuth())
|
||||
|
||||
assert isinstance(response, Response)
|
||||
assert response.status_code == 429
|
||||
body: Final = json.loads(response.body)
|
||||
assert body["type"] == "error"
|
||||
assert body["error"]["type"] == "rate_limit_error"
|
||||
assert [json.loads(call.request.content)["model"] for call in messages_route.calls] == ["claude-sonnet-5-5"]
|
||||
|
||||
|
||||
class TestEventLoggingBatchEndpoint:
|
||||
"""Test the stubbed event logging batch endpoint"""
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ load_dotenv()
|
|||
# this file is to test litellm/proxy
|
||||
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi import Request, Response
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.types import Message
|
||||
|
||||
|
|
@ -386,7 +386,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
|
||||
try:
|
||||
# Call the endpoint
|
||||
response = await count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
# Verify response format matches Anthropic spec
|
||||
assert isinstance(response, dict)
|
||||
|
|
@ -466,7 +466,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
|
||||
try:
|
||||
# Call the endpoint
|
||||
response = await count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
# Verify response format matches Anthropic spec
|
||||
assert isinstance(response, dict)
|
||||
|
|
@ -574,7 +574,7 @@ async def test_anthropic_endpoint_error_handling():
|
|||
anthropic_endpoints.read_request_body = mock_read_request_body
|
||||
|
||||
try:
|
||||
response = await count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response = await count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert json.loads(response.body) == {
|
||||
|
|
@ -615,6 +615,8 @@ async def test_factory_anthropic_endpoint_calls_anthropic_counter():
|
|||
}
|
||||
]
|
||||
|
||||
mock_router.get_model_list.return_value = mock_router.model_list
|
||||
|
||||
# Mock the async method properly
|
||||
mock_router.async_get_available_deployment = AsyncMock(
|
||||
return_value={
|
||||
|
|
@ -676,6 +678,8 @@ async def test_factory_gpt4_endpoint_does_not_call_anthropic_counter():
|
|||
}
|
||||
]
|
||||
|
||||
mock_router.get_model_list.return_value = mock_router.model_list
|
||||
|
||||
# Mock the async method properly
|
||||
mock_router.async_get_available_deployment = AsyncMock(
|
||||
return_value={
|
||||
|
|
@ -1273,7 +1277,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
proxy_server.count_request_tokens = mock_token_counter_error
|
||||
|
||||
try:
|
||||
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 400
|
||||
|
|
@ -1330,7 +1334,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
proxy_server.count_request_tokens = mock_token_counter_error
|
||||
|
||||
try:
|
||||
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 403
|
||||
|
|
@ -1387,7 +1391,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
proxy_server.count_request_tokens = mock_token_counter_error
|
||||
|
||||
try:
|
||||
response: Final[object] = await anthropic_count_tokens(mock_request, mock_user_api_key_dict)
|
||||
response: Final[object] = await anthropic_count_tokens(Response(), mock_request, mock_user_api_key_dict)
|
||||
|
||||
assert isinstance(response, JSONResponse)
|
||||
assert response.status_code == 429
|
||||
|
|
@ -1463,7 +1467,7 @@ async def test_anthropic_count_tokens_drops_a_malformed_tool_result_text_when_dr
|
|||
route: Final = respx.post("https://api.openai.com/v1/responses/input_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 9})
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
|
||||
|
||||
assert response["input_tokens"] == 9
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
|
|
@ -1508,7 +1512,7 @@ async def _count_on_a_model_without_a_count_api(
|
|||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "proxy_config", proxy_server.ProxyConfig())
|
||||
with respx.mock:
|
||||
return await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
return await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1589,7 +1593,7 @@ async def test_anthropic_count_tokens_on_a_claude_deployment_drops_a_malformed_t
|
|||
route: Final = respx.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 7})
|
||||
)
|
||||
response: Final = await anthropic_count_tokens(request, UserAPIKeyAuth())
|
||||
response: Final = await anthropic_count_tokens(Response(), request, UserAPIKeyAuth())
|
||||
|
||||
assert response["input_tokens"] == 7
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
|
|
@ -3033,7 +3037,7 @@ async def _count_through_the_messages_endpoint(
|
|||
return_value=AsyncHTTPHandler(transport=httpx.MockTransport(peer)),
|
||||
),
|
||||
):
|
||||
response: Final = await anthropic_count_tokens(mock_request, MagicMock())
|
||||
response: Final = await anthropic_count_tokens(Response(), mock_request, MagicMock())
|
||||
return response, peer.request_bodies
|
||||
|
||||
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -47183,7 +47183,7 @@ export interface components {
|
|||
* Litellm Count Source
|
||||
* @enum {string}
|
||||
*/
|
||||
litellm_count_source?: "provider" | "token_count_model" | "local_estimate";
|
||||
litellm_count_source?: "provider" | "token_count_model" | "local_estimate" | "billed_request";
|
||||
/** Prompttokensdetails */
|
||||
promptTokensDetails: components["schemas"]["PromptTokensDetails"][];
|
||||
/** Totaltokens */
|
||||
|
|
@ -47213,7 +47213,7 @@ export interface components {
|
|||
/** TokenCountResponse */
|
||||
TokenCountResponse: {
|
||||
/** Count Source */
|
||||
count_source?: ("provider" | "token_count_model" | "local_estimate") | null;
|
||||
count_source?: ("provider" | "token_count_model" | "local_estimate" | "billed_request") | null;
|
||||
/**
|
||||
* Error
|
||||
* @default false
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue