mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 1c109685ef into 3b8ad37296
This commit is contained in:
commit
da876d8d0c
18 changed files with 738 additions and 60 deletions
|
|
@ -9,6 +9,7 @@ import sys
|
|||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TextIO
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
|
@ -755,17 +756,20 @@ class JsonFormatter(Formatter):
|
|||
# fields, and a message that merely happens to parse as JSON/dict (e.g. a proxy
|
||||
# log line dumping raw request headers) must never be able to claim them, even on
|
||||
# a record the filter hasn't stamped yet (no correlation context active for it).
|
||||
extras: Final = MappingProxyType(
|
||||
{key: value for key, value in _record_attributes(record).items() if key not in _NON_EXTRA_RECORD_ATTRS}
|
||||
)
|
||||
parsed = _try_parse_json_message(message_str)
|
||||
if parsed is None:
|
||||
parsed = _try_parse_embedded_python_dict(message_str)
|
||||
if parsed is not None:
|
||||
for key, value in parsed.items():
|
||||
if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS:
|
||||
if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS and key not in extras:
|
||||
json_record[key] = value
|
||||
|
||||
# Include extra attributes passed via logger.debug("msg", extra={...})
|
||||
for key, value in _record_attributes(record).items():
|
||||
if key not in _NON_EXTRA_RECORD_ATTRS and key not in json_record:
|
||||
for key, value in extras.items():
|
||||
if key not in json_record:
|
||||
json_record[key] = value
|
||||
|
||||
# trace_id/session_id are reserved: CorrelationContextFilter is the only
|
||||
|
|
|
|||
|
|
@ -206,6 +206,10 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
encoded_body: Final = base64.b64encode(json.dumps(body_data).encode()).decode()
|
||||
return {"input": {"invokeModel": {"body": encoded_body}}}
|
||||
|
||||
@staticmethod
|
||||
def count_tokens_model_id(model: str) -> str:
|
||||
return get_bedrock_base_model(model).removeprefix("bedrock/")
|
||||
|
||||
def get_bedrock_count_tokens_endpoint(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -225,12 +229,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
|
|||
Returns:
|
||||
Complete endpoint URL for CountTokens API
|
||||
"""
|
||||
# Use existing LiteLLM function to get the base model ID (removes region prefix)
|
||||
model_id = get_bedrock_base_model(model)
|
||||
|
||||
# Remove bedrock/ prefix if present
|
||||
model_id = model_id.removeprefix("bedrock/") # Remove "bedrock/" prefix
|
||||
encoded_model_id: Final = self.encode_model_id(model_id=model_id)
|
||||
encoded_model_id: Final = self.encode_model_id(model_id=self.count_tokens_model_id(model))
|
||||
|
||||
base_url, _ = self.get_runtime_endpoint(
|
||||
api_base=api_base,
|
||||
|
|
|
|||
|
|
@ -3461,12 +3461,12 @@ class LiteLLM_SpendLogs_ResponseObject(LiteLLMPydanticObjectBase):
|
|||
class TokenCountRequest(LiteLLMPydanticObjectBase):
|
||||
model: str
|
||||
prompt: str | None = None
|
||||
messages: list[dict] | None = None
|
||||
messages: list[dict[Any, Any]] | None = None
|
||||
"""
|
||||
Anthropic token counting endpoint uses /messages
|
||||
"""
|
||||
|
||||
contents: list[dict] | None = None
|
||||
contents: list[dict[Any, Any]] | None = None
|
||||
"""
|
||||
Google /countTokens endpoint expects contents to be a list of dicts with the following structure:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -411,6 +411,7 @@ async def count_tokens(
|
|||
request=token_request,
|
||||
call_endpoint=True,
|
||||
messages_format="anthropic",
|
||||
litellm_call_id=litellm_call_id,
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except HTTPException as e:
|
||||
|
|
|
|||
116
litellm/proxy/common_utils/token_count_log.py
Normal file
116
litellm/proxy/common_utils/token_count_log.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
||||
from fastapi import status
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.common_utils.openai_error_payload import error_status_code
|
||||
from litellm.types.token_count import TokenCountSource
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
||||
_BARE_LOGFMT_VALUE: Final = re.compile(r"[\w.:/@+-]+", re.ASCII)
|
||||
_JSON_ESCAPED_BRACES: Final = str.maketrans({"{": "\\u007b", "}": "\\u007d"})
|
||||
|
||||
|
||||
def _logfmt_value(value: str | None) -> str:
|
||||
if value is None:
|
||||
return "None"
|
||||
if _BARE_LOGFMT_VALUE.fullmatch(value):
|
||||
return value
|
||||
return json.dumps(value).translate(_JSON_ESCAPED_BRACES)
|
||||
|
||||
|
||||
_Counted: Final = TypeVar("_Counted")
|
||||
_ProviderCount: Final = TypeVar("_ProviderCount", bound=Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CountFields:
|
||||
model_used: str | None
|
||||
input_tokens: int | None
|
||||
count_source: TokenCountSource | None
|
||||
|
||||
|
||||
_FAILED_COUNT: Final = _CountFields(model_used=None, input_tokens=None, count_source=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _CountRequest:
|
||||
litellm_call_id: str
|
||||
request_model: str
|
||||
model_id: str | None
|
||||
|
||||
|
||||
def deployment_model_id(deployment: Mapping[str, object] | None) -> str | None:
|
||||
match deployment:
|
||||
case {"model_info": {"id": str() as model_id}}:
|
||||
return model_id
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _log_token_count(request: _CountRequest, fields: _CountFields, status_code: int) -> None:
|
||||
verbose_proxy_logger.info(
|
||||
"token count - litellm_call_id=%s request_model=%s model_id=%s model_used=%s input_tokens=%s "
|
||||
"count_source=%s status_code=%s",
|
||||
_logfmt_value(request.litellm_call_id),
|
||||
_logfmt_value(request.request_model),
|
||||
_logfmt_value(request.model_id),
|
||||
_logfmt_value(fields.model_used),
|
||||
fields.input_tokens,
|
||||
fields.count_source,
|
||||
status_code,
|
||||
extra=MappingProxyType(
|
||||
{
|
||||
"litellm_call_id": request.litellm_call_id,
|
||||
"request_model": request.request_model,
|
||||
"model_id": request.model_id,
|
||||
"model_used": fields.model_used,
|
||||
"input_tokens": fields.input_tokens,
|
||||
"count_source": fields.count_source,
|
||||
"status_code": status_code,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _logged(
|
||||
count: Awaitable[_Counted], request: _CountRequest, fields_of: Callable[[_Counted], _CountFields]
|
||||
) -> _Counted:
|
||||
try:
|
||||
counted: Final = await count
|
||||
except Exception as e:
|
||||
_log_token_count(request, _FAILED_COUNT, error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR))
|
||||
raise
|
||||
_log_token_count(request, fields_of(counted), status.HTTP_200_OK)
|
||||
return counted
|
||||
|
||||
|
||||
def _token_count_response_fields(counted: TokenCountResponse) -> _CountFields:
|
||||
return _CountFields(
|
||||
model_used=counted.model_used, input_tokens=counted.total_tokens, count_source=counted.count_source
|
||||
)
|
||||
|
||||
|
||||
async def logged_token_count(
|
||||
count: Awaitable[TokenCountResponse], litellm_call_id: str, request_model: str, model_id: str | None
|
||||
) -> TokenCountResponse:
|
||||
return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), _token_count_response_fields)
|
||||
|
||||
|
||||
async def logged_provider_token_count(
|
||||
count: Awaitable[_ProviderCount], litellm_call_id: str, request_model: str, model_id: str | None, model_used: str
|
||||
) -> _ProviderCount:
|
||||
def provider_fields(counted: _ProviderCount) -> _CountFields:
|
||||
input_tokens: Final = counted.get("input_tokens")
|
||||
return _CountFields(
|
||||
model_used=model_used,
|
||||
input_tokens=input_tokens if isinstance(input_tokens, int) and not isinstance(input_tokens, bool) else None,
|
||||
count_source="provider",
|
||||
)
|
||||
|
||||
return await _logged(count, _CountRequest(litellm_call_id, request_model, model_id), provider_fields)
|
||||
|
|
@ -21,6 +21,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
read_request_body,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
|
||||
|
||||
|
|
@ -286,6 +287,7 @@ async def google_count_tokens(request: Request, model_name: str):
|
|||
from litellm.proxy.common_utils.http_parsing_utils import read_request_body
|
||||
from litellm.proxy.proxy_server import token_counter as internal_token_counter
|
||||
|
||||
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER))
|
||||
data: Final = await read_request_body(request=request)
|
||||
try:
|
||||
token_request: Final = _google_count_tokens_request(model_name, data)
|
||||
|
|
@ -298,13 +300,15 @@ async def google_count_tokens(request: Request, model_name: str):
|
|||
)
|
||||
|
||||
try:
|
||||
token_response: Final = await internal_token_counter(request=token_request, call_endpoint=True)
|
||||
token_response: Final = await internal_token_counter(
|
||||
request=token_request, call_endpoint=True, litellm_call_id=litellm_call_id
|
||||
)
|
||||
except ProxyException as error:
|
||||
return _google_error_response(request, error)
|
||||
except HTTPException as error:
|
||||
return _google_error_response(request, proxy_exception_from_http_exception(error, {}))
|
||||
except Exception as error:
|
||||
log_llm_api_exception(error, resolve_litellm_call_id(request.headers.get("x-litellm-call-id")))
|
||||
log_llm_api_exception(error, litellm_call_id)
|
||||
return _google_error_response(
|
||||
request,
|
||||
ProxyException(message="Internal server error", type="internal_server_error", param=None, code=500),
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ from litellm.proxy.common_request_processing import (
|
|||
ProxyBaseLLMRequestProcessing,
|
||||
model_deployments_support_stream_options,
|
||||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -99,10 +100,12 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
safe_get_request_headers,
|
||||
safe_set_request_parsed_body,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import LITELLM_CALL_ID_HEADER
|
||||
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
wrap_passthrough_sse_bytes_with_keepalive_pings,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_provider_token_count
|
||||
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
|
||||
from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
|
||||
stream_usage_relay,
|
||||
|
|
@ -1272,34 +1275,35 @@ async def handle_bedrock_count_tokens(
|
|||
litellm_params: Final = {"user_api_key_dict": user_api_key_dict}
|
||||
resolved_model = model # Default fallback
|
||||
|
||||
if llm_router:
|
||||
deployments: Final = llm_router.get_model_list(model_name=model)
|
||||
if deployments and len(deployments) > 0:
|
||||
# Get the first matching deployment
|
||||
deployment: Final = deployments[0]
|
||||
model_litellm_params: Final = deployment.get("litellm_params", {})
|
||||
deployments: Final = llm_router.get_model_list(model_name=model) if llm_router else None
|
||||
deployment: Final = deployments[0] if deployments else None
|
||||
if deployment is not None:
|
||||
model_litellm_params: Final = deployment.get("litellm_params", {})
|
||||
|
||||
# Get the resolved model ID from the configuration
|
||||
if "model" in model_litellm_params:
|
||||
resolved_model = model_litellm_params["model"]
|
||||
# Get the resolved model ID from the configuration
|
||||
if "model" in model_litellm_params:
|
||||
resolved_model = model_litellm_params["model"]
|
||||
|
||||
# Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery
|
||||
for key, value in model_litellm_params.items():
|
||||
if key != "user_api_key_dict": # Don't overwrite user_api_key_dict
|
||||
litellm_params[key] = value
|
||||
# Copy all litellm_params - BaseAWSLLM will handle AWS credential discovery
|
||||
for key, value in model_litellm_params.items():
|
||||
if key != "user_api_key_dict": # Don't overwrite user_api_key_dict
|
||||
litellm_params[key] = value
|
||||
|
||||
verbose_proxy_logger.debug("Count tokens litellm_params: %s", litellm_params)
|
||||
verbose_proxy_logger.debug("Resolved model: %s", resolved_model)
|
||||
|
||||
# Handle the count tokens request
|
||||
result: Final = await handler.handle_count_tokens_request(
|
||||
request_data=request_body,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
return await logged_provider_token_count(
|
||||
handler.handle_count_tokens_request(
|
||||
request_data=request_body,
|
||||
litellm_params=litellm_params,
|
||||
resolved_model=resolved_model,
|
||||
),
|
||||
litellm_call_id=resolve_litellm_call_id(request.headers.get(LITELLM_CALL_ID_HEADER)),
|
||||
request_model=model,
|
||||
model_id=deployment_model_id(deployment),
|
||||
model_used=handler.count_tokens_model_id(resolved_model),
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
except BedrockError as e:
|
||||
# Convert BedrockError to HTTPException for FastAPI
|
||||
verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e)
|
||||
|
|
|
|||
|
|
@ -552,6 +552,7 @@ from litellm.proxy.common_utils.timezone_utils import (
|
|||
get_budget_reset_settings,
|
||||
get_budget_reset_time,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_log import deployment_model_id, logged_token_count
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
end_user_cache_key,
|
||||
|
|
@ -15244,16 +15245,21 @@ def _system_message(system: object) -> ChatCompletionSystemMessage | None:
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=TokenCountResponse,
|
||||
)
|
||||
async def token_counter(request: TokenCountRequest, call_endpoint: bool = False):
|
||||
async def token_counter(
|
||||
request: TokenCountRequest,
|
||||
call_endpoint: bool = False,
|
||||
litellm_call_id: Annotated[str | None, Header(alias=LITELLM_CALL_ID_HEADER)] = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
request: TokenCountRequest
|
||||
call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs.
|
||||
litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs
|
||||
|
||||
Returns:
|
||||
TokenCountResponse
|
||||
"""
|
||||
return await count_request_tokens(request=request, call_endpoint=call_endpoint)
|
||||
return await count_request_tokens(request=request, call_endpoint=call_endpoint, litellm_call_id=litellm_call_id)
|
||||
|
||||
|
||||
async def count_request_tokens(
|
||||
|
|
@ -15264,6 +15270,53 @@ async def count_request_tokens(
|
|||
context_management: JsonValue = None,
|
||||
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
responses_request: Mapping[str, object] | None = None,
|
||||
litellm_call_id: str | None = None,
|
||||
) -> TokenCountResponse:
|
||||
if request.prompt is None and request.messages is None and request.contents is None:
|
||||
raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided")
|
||||
routed: Final = await _routed_count_deployment(request.model, responses_request)
|
||||
return await logged_token_count(
|
||||
_count_request_tokens(
|
||||
request=request,
|
||||
routed=routed,
|
||||
call_endpoint=call_endpoint,
|
||||
messages_format=messages_format,
|
||||
output_format=output_format,
|
||||
context_management=context_management,
|
||||
request_litellm_params=request_litellm_params,
|
||||
responses_request=responses_request,
|
||||
),
|
||||
litellm_call_id=resolve_litellm_call_id(litellm_call_id),
|
||||
request_model=request.model,
|
||||
model_id=deployment_model_id(routed),
|
||||
)
|
||||
|
||||
|
||||
async def _routed_count_deployment(
|
||||
model: str, responses_request: Mapping[str, object] | None
|
||||
) -> Mapping[str, object] | None:
|
||||
if llm_router is None:
|
||||
return None
|
||||
try:
|
||||
return await llm_router.async_get_available_deployment(
|
||||
model=model, request_kwargs=dict(_responses_routing_kwargs(responses_request))
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _count_request_tokens(
|
||||
request: TokenCountRequest,
|
||||
routed: Mapping[str, object] | None,
|
||||
call_endpoint: bool,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
output_format: JsonValue,
|
||||
context_management: JsonValue,
|
||||
request_litellm_params: Mapping[str, JsonValue],
|
||||
responses_request: Mapping[str, object] | None,
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15271,26 +15324,9 @@ async def count_request_tokens(
|
|||
tools: Final = request.tools
|
||||
system: Final = request.system
|
||||
|
||||
#########################################################
|
||||
# Validate request
|
||||
#########################################################
|
||||
if prompt is None and messages is None and contents is None:
|
||||
raise HTTPException(status_code=400, detail="prompt or messages or contents must be provided")
|
||||
|
||||
deployment: dict[str, Any] | None = None
|
||||
deployment: dict[str, Any] | None = None if routed is None else dict(routed)
|
||||
litellm_model_name = None
|
||||
model_info: ModelMapInfo | None = None
|
||||
if llm_router is not None:
|
||||
# get 1 deployment corresponding to the model
|
||||
try:
|
||||
deployment = await llm_router.async_get_available_deployment(
|
||||
model=request.model,
|
||||
request_kwargs=dict(_responses_routing_kwargs(responses_request)),
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"litellm.proxy.proxy_server.token_counter(): Exception occured while getting deployment"
|
||||
)
|
||||
if deployment is not None:
|
||||
litellm_model_name = deployment.get("litellm_params", {}).get("model")
|
||||
model_info = deployment.get("model_info", {})
|
||||
|
|
|
|||
|
|
@ -1407,7 +1407,12 @@ class _TokenCountPayload(TypedDict):
|
|||
|
||||
class _TokenCounter(Protocol):
|
||||
def __call__(
|
||||
self, *, request: TokenCountRequest, call_endpoint: bool, responses_request: Mapping[str, object] | None
|
||||
self,
|
||||
*,
|
||||
request: TokenCountRequest,
|
||||
call_endpoint: bool,
|
||||
responses_request: Mapping[str, object] | None,
|
||||
litellm_call_id: str | None,
|
||||
) -> Awaitable[TokenCountResponse]: ...
|
||||
|
||||
|
||||
|
|
@ -1700,7 +1705,10 @@ async def responses_input_tokens(
|
|||
)
|
||||
try:
|
||||
token_response: Final = await token_counter(
|
||||
request=token_request, call_endpoint=True, responses_request=responses_request
|
||||
request=token_request,
|
||||
call_endpoint=True,
|
||||
responses_request=responses_request,
|
||||
litellm_call_id=request.headers.get(LITELLM_CALL_ID_HEADER),
|
||||
)
|
||||
except ProxyException as e:
|
||||
if e.type != _PROVIDER_COUNT_ERROR_TYPE:
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objec
|
|||
import json
|
||||
import logging
|
||||
import unittest
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
|
@ -615,6 +617,104 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e
|
|||
assert counter.await_args.kwargs["request"].tools is None
|
||||
|
||||
|
||||
def _count_on_claude(
|
||||
monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str]
|
||||
) -> httpx.Response:
|
||||
from fastapi import FastAPI
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import router
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"}
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "count-log-claude",
|
||||
"litellm_params": {**claude, "order": 2},
|
||||
"model_info": {"id": "count-log-fallback-deployment"},
|
||||
},
|
||||
{
|
||||
"model_name": "count-log-claude",
|
||||
"litellm_params": {**claude, "order": 1},
|
||||
"model_info": {"id": "count-log-routed-deployment"},
|
||||
},
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
anthropic_count_tokens_handler,
|
||||
"get_async_httpx_client",
|
||||
lambda *args, **kwargs: AsyncHTTPHandler(transport=httpx.MockTransport(peer)),
|
||||
)
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth()
|
||||
return TestClient(app).post(
|
||||
"/v1/messages/count_tokens",
|
||||
json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
headers=dict(headers),
|
||||
)
|
||||
|
||||
|
||||
def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
response: Final = _count_on_claude(
|
||||
monkeypatch,
|
||||
lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request),
|
||||
{"x-litellm-call-id": "count-log-msg-claude"},
|
||||
)
|
||||
|
||||
assert response.json() == {"input_tokens": 77, "litellm_count_source": "provider"}, response.text
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-msg-claude",
|
||||
"request_model": "count-log-claude",
|
||||
"model_id": "count-log-routed-deployment",
|
||||
"model_used": "claude-zeta-9",
|
||||
"input_tokens": 77,
|
||||
"count_source": "provider",
|
||||
"status_code": 200,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
rejection: Final = {
|
||||
"type": "error",
|
||||
"error": {"type": "invalid_request_error", "message": "messages.0.content.0: invalid base64 data"},
|
||||
}
|
||||
|
||||
response: Final = _count_on_claude(
|
||||
monkeypatch,
|
||||
lambda request: httpx.Response(400, json=rejection, request=request),
|
||||
{"x-litellm-call-id": "count-log-msg-error"},
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json()["error"] == rejection["error"]
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-msg-error",
|
||||
"request_model": "count-log-claude",
|
||||
"model_id": "count-log-routed-deployment",
|
||||
"model_used": None,
|
||||
"input_tokens": None,
|
||||
"count_source": None,
|
||||
"status_code": 400,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestEventLoggingBatchEndpoint:
|
||||
"""Test the stubbed event logging batch endpoint"""
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
|
|
@ -17,6 +18,7 @@ from prisma.errors import ClientNotConnectedError
|
|||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.integrations.otel.model.payloads import ServiceSpanData
|
||||
from litellm.integrations.otel.model.spans import service_span_name
|
||||
|
|
@ -424,6 +426,31 @@ def fresh_agent_read_through(monkeypatch):
|
|||
return read_through
|
||||
|
||||
|
||||
TOKEN_COUNT_LOG_FIELDS: Final = (
|
||||
"litellm_call_id",
|
||||
"request_model",
|
||||
"model_id",
|
||||
"model_used",
|
||||
"input_tokens",
|
||||
"count_source",
|
||||
"status_code",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def token_count_log_lines(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> Iterator[Callable[[], tuple[dict[str, object], ...]]]:
|
||||
"""The structured fields of every INFO token count line the proxy logger writes while the test runs."""
|
||||
monkeypatch.setattr(verbose_proxy_logger, "propagate", True)
|
||||
with caplog.at_level(logging.INFO, logger=verbose_proxy_logger.name):
|
||||
yield lambda: tuple(
|
||||
{field: record.__dict__[field] for field in TOKEN_COUNT_LOG_FIELDS}
|
||||
for record in caplog.records
|
||||
if record.levelno == logging.INFO and "count_source" in record.__dict__
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]:
|
||||
"""The ``postgres.{verb} {table}`` names OTel would render for every DB service event
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ async def test_proxy_gemini_to_openai_like_model_token_counting():
|
|||
request=Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"headers": [],
|
||||
"parsed_body": (
|
||||
["contents"],
|
||||
{"contents": [{"parts": [{"text": "Hello, how are you?"}]}]},
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
Test to verify the Google GenAI proxy API endpoints
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.types.utils import TokenCountResponse
|
||||
|
|
@ -240,3 +242,56 @@ def test_google_count_tokens_uses_normalized_total_when_provider_shape_differs()
|
|||
assert body["promptTokensDetails"] == []
|
||||
assert body["litellm_count_source"] == "token_count_model"
|
||||
assert body["litellm_count_model"] == "us.anthropic.claude-opus-5-5"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1beta/models/count-log-claude:countTokens", "/models/count-log-claude:countTokens"])
|
||||
def test_google_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]], path: str
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm import Router
|
||||
from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
claude: Final = {"model": "anthropic/claude-zeta-9", "api_key": "test-key", "api_base": "https://count-tokens.test"}
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "count-log-claude",
|
||||
"litellm_params": claude,
|
||||
"model_info": {"id": "count-log-gemini-deployment"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
anthropic_count_tokens_handler,
|
||||
"get_async_httpx_client",
|
||||
lambda *args, **kwargs: AsyncHTTPHandler(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, json={"input_tokens": 77}, request=request)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
response: Final = _build_test_client().post(
|
||||
path,
|
||||
json={"contents": [{"role": "user", "parts": [{"text": "Hello"}]}]},
|
||||
headers={"x-litellm-call-id": "count-log-gemini"},
|
||||
)
|
||||
|
||||
assert (response.status_code, response.json()["totalTokens"]) == (200, 77), response.text
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-gemini",
|
||||
"request_model": "count-log-claude",
|
||||
"model_id": "count-log-gemini-deployment",
|
||||
"model_used": "claude-zeta-9",
|
||||
"input_tokens": 77,
|
||||
"count_source": "provider",
|
||||
"status_code": 200,
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2142,7 +2142,6 @@ class TestBedrockPassthroughAwsErrors:
|
|||
assert exc_info.value.aws_error_type == "ValidationException"
|
||||
assert (exc_info.value.error.message, exc_info.value.error.code) == (message, status_code)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_request_check_is_raised_with_the_aws_error_type_it_carries(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -2183,6 +2182,7 @@ class TestBedrockPassthroughAwsErrors:
|
|||
await self._answer(error)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.headers.get("x-model-group")) == ("403", "modèle")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("error", "message", "code"),
|
||||
|
|
@ -7250,6 +7250,107 @@ async def test_bedrock_count_tokens_error_forwards_provider_headers():
|
|||
assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-count-tokens-500"
|
||||
|
||||
|
||||
_BEDROCK_COUNT_ALIAS: Final = "count-log-bedrock"
|
||||
_BEDROCK_COUNT_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
|
||||
|
||||
async def _bedrock_native_count(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, aws_reply: httpx.Response
|
||||
) -> dict[str, object]:
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
handle_bedrock_count_tokens,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": _BEDROCK_COUNT_ALIAS,
|
||||
"litellm_params": {
|
||||
"model": _BEDROCK_COUNT_MODEL,
|
||||
"aws_region_name": "us-east-1",
|
||||
"aws_access_key_id": "AKIATESTACCESSKEY",
|
||||
"aws_secret_access_key": "test-secret",
|
||||
},
|
||||
"model_info": {"id": "count-log-bedrock-deployment"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com", path__regex=r"/count-tokens$").mock(
|
||||
return_value=aws_reply
|
||||
)
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": f"/bedrock/model/{_BEDROCK_COUNT_ALIAS}/count-tokens",
|
||||
"headers": [(b"x-litellm-call-id", b"bedrock-native-count")],
|
||||
}
|
||||
)
|
||||
return await handle_bedrock_count_tokens(
|
||||
endpoint=f"model/{_BEDROCK_COUNT_ALIAS}/count-tokens",
|
||||
request=request,
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=MagicMock(),
|
||||
request_body={"model": _BEDROCK_COUNT_ALIAS, "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_native_count_tokens_logs_the_count_against_the_router_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
httpx_transport: None,
|
||||
respx_mock: respx.MockRouter,
|
||||
token_count_log_lines: Callable[[], tuple[dict[str, object], ...]],
|
||||
) -> None:
|
||||
result: Final = await _bedrock_native_count(
|
||||
monkeypatch, respx_mock, httpx.Response(200, json={"inputTokens": 12})
|
||||
)
|
||||
|
||||
assert result == {"input_tokens": 12}
|
||||
assert len(respx_mock.calls) == 1
|
||||
assert respx_mock.calls.last.request.url.path == "/model/anthropic.claude-haiku-4-5-20251001-v1:0/count-tokens"
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "bedrock-native-count",
|
||||
"request_model": _BEDROCK_COUNT_ALIAS,
|
||||
"model_id": "count-log-bedrock-deployment",
|
||||
"model_used": "anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"input_tokens": 12,
|
||||
"count_source": "provider",
|
||||
"status_code": 200,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_native_count_tokens_logs_the_provider_status_when_the_count_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
httpx_transport: None,
|
||||
respx_mock: respx.MockRouter,
|
||||
token_count_log_lines: Callable[[], tuple[dict[str, object], ...]],
|
||||
) -> None:
|
||||
rejected: Final = httpx.Response(400, json={"message": "The provided model doesn't support counting tokens."})
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await _bedrock_native_count(monkeypatch, respx_mock, rejected)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "bedrock-native-count",
|
||||
"request_model": _BEDROCK_COUNT_ALIAS,
|
||||
"model_id": "count-log-bedrock-deployment",
|
||||
"model_used": None,
|
||||
"input_tokens": None,
|
||||
"count_source": None,
|
||||
"status_code": 400,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class _AzureGroupRouter:
|
||||
def __init__(self, captured: list[dict]) -> None:
|
||||
self.captured = captured
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import contextlib
|
|||
import dataclasses
|
||||
import json
|
||||
import unittest
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any, Final, Literal, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ from fastapi.testclient import TestClient
|
|||
from httpx import Response
|
||||
|
||||
import litellm
|
||||
from litellm.constants import EMPTY_MAPPING
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
|
||||
|
|
@ -4275,6 +4276,7 @@ class TestResponsesInputTokens:
|
|||
llm_router: litellm.Router | None = None,
|
||||
response_id_security: object | None = None,
|
||||
managed_files: object | None = None,
|
||||
headers: Mapping[str, str] = EMPTY_MAPPING,
|
||||
) -> tuple[Response, AsyncMock]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
@ -4307,7 +4309,7 @@ class TestResponsesInputTokens:
|
|||
app.dependency_overrides[_proxy_managed_files_hook] = lambda: managed_files
|
||||
try:
|
||||
client = TestClient(app)
|
||||
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876"})
|
||||
response = client.post(path, json=body, headers={"Authorization": "Bearer sk-9876", **headers})
|
||||
return response, token_counter_mock
|
||||
finally:
|
||||
app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
|
@ -4552,6 +4554,63 @@ class TestResponsesInputTokens:
|
|||
assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text
|
||||
assert sent == [{"model": "gpt-4o-mini", "input": "hi"}]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"]
|
||||
)
|
||||
def test_count_logs_one_line_with_the_callers_call_id_model_and_count(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
token_count_log_lines: Callable[[], tuple[dict[str, object], ...]],
|
||||
path: str,
|
||||
) -> None:
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.openai.responses.count_tokens import handler as openai_count_tokens_http
|
||||
from litellm.proxy.proxy_server import count_request_tokens
|
||||
|
||||
gpt: Final = {"model": "openai/gpt-4o-mini", "api_key": "test-key", "api_base": "https://count.test/v1"}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "count-log-gpt",
|
||||
"litellm_params": gpt,
|
||||
"model_info": {"id": "count-log-responses-deployment"},
|
||||
}
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(
|
||||
openai_count_tokens_http,
|
||||
"get_async_httpx_client",
|
||||
lambda *args, **kwargs: AsyncHTTPHandler(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200, json={"object": "response.input_tokens", "input_tokens": 7}, request=request
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
response, _ = self._post_input_tokens(
|
||||
{"model": "count-log-gpt", "input": "hi"},
|
||||
path=path,
|
||||
counter=AsyncMock(side_effect=count_request_tokens),
|
||||
llm_router=router,
|
||||
headers={"x-litellm-call-id": "count-log-responses"},
|
||||
)
|
||||
|
||||
assert (response.status_code, response.json().get("input_tokens")) == (200, 7), response.text
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-responses",
|
||||
"request_model": "count-log-gpt",
|
||||
"model_id": "count-log-responses-deployment",
|
||||
"model_used": "gpt-4o-mini",
|
||||
"input_tokens": 7,
|
||||
"count_source": "provider",
|
||||
"status_code": 200,
|
||||
},
|
||||
)
|
||||
|
||||
def test_a_managed_file_the_key_cannot_read_answers_403_without_a_count(self) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import json
|
|||
import logging
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
|
@ -40,7 +41,7 @@ from starlette.types import Message
|
|||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._logging import JsonFormatter, verbose_proxy_logger
|
||||
from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
|
@ -364,6 +365,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -444,6 +446,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -1262,6 +1265,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1319,6 +1323,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1376,6 +1381,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
litellm_call_id=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
@ -3892,3 +3898,139 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r
|
|||
]
|
||||
assert [sorted(message) for message in messages if isinstance(message, dict)] == [["content", "role"]] * 3
|
||||
spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form"))
|
||||
|
||||
|
||||
_COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]}
|
||||
)
|
||||
|
||||
|
||||
def _utils_token_counter(
|
||||
monkeypatch: pytest.MonkeyPatch, body: Mapping[str, JsonValue], headers: Mapping[str, str]
|
||||
) -> httpx.Response:
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm.proxy.proxy_server,
|
||||
"llm_router",
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "count-log-gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
|
||||
"model_info": {"id": "count-log-utils-deployment"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-test"))
|
||||
return TestClient(app).post("/utils/token_counter", json=dict(body), headers=dict(headers))
|
||||
|
||||
|
||||
def test_utils_token_counter_logs_one_line_with_the_callers_call_id_model_and_count(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": "count-log-utils"})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-utils",
|
||||
"request_model": "count-log-gpt",
|
||||
"model_id": "count-log-utils-deployment",
|
||||
"model_used": "gpt-4o-mini",
|
||||
"input_tokens": response.json()["total_tokens"],
|
||||
"count_source": "local_estimate",
|
||||
"status_code": 200,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_utils_token_counter_logs_a_generated_call_id_when_the_client_sends_none(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
response: Final = _utils_token_counter(monkeypatch, _COUNTED_HELLO, {})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
(line,) = token_count_log_lines()
|
||||
assert str(uuid.UUID(str(line["litellm_call_id"]))) == line["litellm_call_id"]
|
||||
|
||||
|
||||
def test_utils_token_counter_failed_count_logs_its_status_in_place_of_a_count(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
response: Final = _utils_token_counter(
|
||||
monkeypatch, {**_COUNTED_HELLO, "prompt": "and a prompt"}, {"x-litellm-call-id": "count-log-utils-error"}
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert response.json() == {
|
||||
"error": {
|
||||
"message": "prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt",
|
||||
"type": "invalid_request_error",
|
||||
"param": "prompt",
|
||||
"code": "400",
|
||||
}
|
||||
}
|
||||
assert token_count_log_lines() == (
|
||||
{
|
||||
"litellm_call_id": "count-log-utils-error",
|
||||
"request_model": "count-log-gpt",
|
||||
"model_id": "count-log-utils-deployment",
|
||||
"model_used": None,
|
||||
"input_tokens": None,
|
||||
"count_source": None,
|
||||
"status_code": 400,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_utils_token_counter_request_with_nothing_to_count_writes_no_line(
|
||||
monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]]
|
||||
) -> None:
|
||||
response: Final = _utils_token_counter(
|
||||
monkeypatch, {"model": "count-log-gpt"}, {"x-litellm-call-id": "count-log-utils-empty"}
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
assert token_count_log_lines() == ()
|
||||
|
||||
|
||||
def test_utils_token_counter_json_line_keeps_its_own_fields_when_the_call_id_is_a_dict(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
token_count_log_lines: Callable[[], tuple[dict[str, object], ...]],
|
||||
) -> None:
|
||||
forged_call_id: Final = "{'component':'forged-comp','logger':'forged.py:1','injected_key':'injected'}"
|
||||
|
||||
_utils_token_counter(monkeypatch, _COUNTED_HELLO, {"x-litellm-call-id": forged_call_id})
|
||||
|
||||
(line,) = token_count_log_lines()
|
||||
(record,) = (record for record in caplog.records if "count_source" in record.__dict__)
|
||||
rendered: Final = json.loads(JsonFormatter().format(record))
|
||||
assert line["litellm_call_id"] == forged_call_id
|
||||
assert {key: rendered.get(key) for key in ("litellm_call_id", "component", "logger", "injected_key")} == {
|
||||
"litellm_call_id": forged_call_id,
|
||||
"component": verbose_proxy_logger.name,
|
||||
"logger": f"{record.filename}:{record.lineno}",
|
||||
"injected_key": None,
|
||||
}
|
||||
|
||||
|
||||
def test_utils_token_counter_line_stays_one_line_when_the_model_carries_a_newline(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
token_count_log_lines: Callable[[], tuple[dict[str, object], ...]],
|
||||
) -> None:
|
||||
forged_model: Final = "count-log-gpt\ntoken count - litellm_call_id=forged input_tokens=1"
|
||||
|
||||
_utils_token_counter(monkeypatch, {**_COUNTED_HELLO, "model": forged_model}, {"x-litellm-call-id": "count log"})
|
||||
|
||||
(line,) = token_count_log_lines()
|
||||
(message,) = (record.getMessage() for record in caplog.records if "count_source" in record.__dict__)
|
||||
assert line["request_model"] == forged_model
|
||||
assert "\n" not in message
|
||||
assert message.startswith(f'token count - litellm_call_id="count log" request_model={json.dumps(forged_model)} ')
|
||||
|
|
|
|||
|
|
@ -329,6 +329,24 @@ def test_json_formatter_extra_component_not_overwritten():
|
|||
assert obj["component"] == "auth-service", f"User-supplied component was overwritten, got {obj['component']!r}"
|
||||
|
||||
|
||||
def test_json_formatter_extra_fields_win_over_a_dict_a_caller_put_in_the_message():
|
||||
record = logging.LogRecord(
|
||||
name="LiteLLM Proxy",
|
||||
level=logging.INFO,
|
||||
pathname="",
|
||||
lineno=0,
|
||||
msg="token count - litellm_call_id=%s input_tokens=%s status_code=%s",
|
||||
args=("{'input_tokens': 0, 'status_code': 200, 'injected': 'yes'}", None, 400),
|
||||
exc_info=None,
|
||||
)
|
||||
record.input_tokens = None
|
||||
record.status_code = 400
|
||||
|
||||
obj = json.loads(JsonFormatter().format(record))
|
||||
|
||||
assert (obj["input_tokens"], obj["status_code"], obj["injected"]) == (None, 400, "yes")
|
||||
|
||||
|
||||
def test_initialize_loggers_with_handler_sets_propagate_false():
|
||||
"""
|
||||
Test that the initialize_loggers_with_handler function sets propagate to False for all loggers
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -19058,6 +19058,7 @@ export interface paths {
|
|||
* @description Args:
|
||||
* request: TokenCountRequest
|
||||
* call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs.
|
||||
* litellm_call_id: the x-litellm-call-id header, logged with the count so it joins the inference call's logs
|
||||
*
|
||||
* Returns:
|
||||
* TokenCountResponse
|
||||
|
|
@ -76716,7 +76717,9 @@ export interface operations {
|
|||
query?: {
|
||||
call_endpoint?: boolean;
|
||||
};
|
||||
header?: never;
|
||||
header?: {
|
||||
"x-litellm-call-id"?: string | null;
|
||||
};
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue