mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
test(bedrock): align backported integration harness with release APIs
This commit is contained in:
Joshua Valluru
parent
71b6509656
commit
0dbf29974c
7 changed files with 257 additions and 1652 deletions
|
|
@ -267,3 +267,25 @@ def team_admin_permissions(gateway: Gateway, fields: Sequence[str]) -> Iterator[
|
|||
yield
|
||||
finally:
|
||||
_set_team_admin_permissions(gateway, [str(field) for field in original] if isinstance(original, list) else ())
|
||||
|
||||
|
||||
def held(read: Callable[[], T], satisfied: Callable[[T], bool], *, holding: float, seconds: float = 10) -> T:
|
||||
"""`eventually`, and then `satisfied` must hold on every read for `holding` more seconds; a miss restarts the hold."""
|
||||
deadline: Final = time.monotonic() + seconds
|
||||
while True:
|
||||
converged: Final = eventually(read, satisfied, seconds=max(deadline - time.monotonic(), 0.0))
|
||||
misses: Final = _misses_within(read, satisfied, holding)
|
||||
if not misses:
|
||||
return converged
|
||||
assert time.monotonic() < deadline, f"State did not hold: {misses[0]!r}"
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def _misses_within(read: Callable[[], T], satisfied: Callable[[T], bool], holding: float) -> tuple[T, ...]:
|
||||
until: Final = time.monotonic() + holding
|
||||
while time.monotonic() < until:
|
||||
time.sleep(0.1)
|
||||
observed: Final = read()
|
||||
if not satisfied(observed):
|
||||
return (observed,)
|
||||
return ()
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
from integration._support.process import owned_proxy
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.mcp import (
|
||||
ENTRY_POINTS,
|
||||
|
|
@ -25,6 +24,7 @@ from integration._support.mcp import (
|
|||
text_result,
|
||||
tool_calls,
|
||||
)
|
||||
from integration._support.process import owned_proxy
|
||||
|
||||
ADD: Final = {"http": "add", "sse": "add", "stdio": "add", "openapi": "getpet"}
|
||||
ARGUMENTS: Final = {"add": {"a": 3, "b": 4}, "getpet": {"petId": "7"}}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -137,13 +137,15 @@ async def test_modern_input_request_uses_existing_elicitation_callback(
|
|||
modern_caller: bool, sampling: bool, elicitation_mode: str
|
||||
) -> None:
|
||||
from queue import SimpleQueue
|
||||
|
||||
from mcp.types import (
|
||||
ElicitResult,
|
||||
ElicitRequestParams,
|
||||
TextContent,
|
||||
CreateMessageRequestParams,
|
||||
CreateMessageResult,
|
||||
ElicitRequestParams,
|
||||
ElicitResult,
|
||||
TextContent,
|
||||
)
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.interactions import BoundInputRequiredResult
|
||||
|
||||
observed: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
|
|
@ -1883,7 +1885,9 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_http_status_check_allows_auth_refresh_before_rejecting() -> None:
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import ClientCredentialsBearerAuth
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.client_credentials import (
|
||||
ClientCredentialsBearerAuth,
|
||||
)
|
||||
|
||||
seen = []
|
||||
|
||||
|
|
@ -2148,6 +2152,7 @@ def test_sse_read_failure_is_preserved() -> None:
|
|||
@pytest.mark.parametrize("mode", ["ok", "closed", "silent"])
|
||||
async def test_transport_completion_and_normal_messages(transport: MCPTransport, mode: str, protocol_version: str) -> None:
|
||||
from mcp import ClientSession
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.rest_endpoints import _connection_error_message
|
||||
|
||||
logging_callback: Final = AsyncMock()
|
||||
|
|
@ -3637,7 +3642,13 @@ def test_modern_upstream_rejects_legacy_sse_transport() -> None:
|
|||
("resources/templates/list", "resourceTemplates", {"name": "second", "uriTemplate": "status://{name}"}),
|
||||
])
|
||||
async def test_single_catalog_page_preserves_cursor_metadata_and_request_cursor(method, field, item):
|
||||
from mcp.types import ListToolsRequest, ListPromptsRequest, ListResourcesRequest, ListResourceTemplatesRequest, PaginatedRequestParams
|
||||
from mcp.types import (
|
||||
ListPromptsRequest,
|
||||
ListResourcesRequest,
|
||||
ListResourceTemplatesRequest,
|
||||
ListToolsRequest,
|
||||
PaginatedRequestParams,
|
||||
)
|
||||
|
||||
def respond(request: httpx2.Request) -> httpx2.Response:
|
||||
if request.method != "POST":
|
||||
|
|
@ -3691,7 +3702,10 @@ async def test_single_catalog_page_preserves_cursor_metadata_and_request_cursor(
|
|||
@pytest.mark.parametrize("cursor", [None, "continuation"])
|
||||
async def test_optional_catalog_distinguishes_absent_capability_from_failed_continuation(method, failure, cursor):
|
||||
from mcp.types import (
|
||||
ListPromptsRequest, ListResourcesRequest, ListResourceTemplatesRequest, PaginatedRequestParams,
|
||||
ListPromptsRequest,
|
||||
ListResourcesRequest,
|
||||
ListResourceTemplatesRequest,
|
||||
PaginatedRequestParams,
|
||||
)
|
||||
|
||||
methods = []
|
||||
|
|
|
|||
|
|
@ -1,21 +1,21 @@
|
|||
import sys
|
||||
import ast, asyncio, os
|
||||
import ast
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from unittest import mock
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
import importlib
|
||||
|
||||
import litellm
|
||||
from litellm import constants, get_llm_provider, MorphChatConfig
|
||||
from litellm import MorphChatConfig, constants, get_llm_provider
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
def _build_constant_env_var_map() -> dict[str, str]:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import datetime
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final, Literal, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -11,7 +13,16 @@ from pydantic import BaseModel
|
|||
|
||||
import litellm
|
||||
from litellm import TranscriptionResponse, model_cost
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
|
||||
from litellm.cost_calculator import (
|
||||
BaseTokenUsageProcessor,
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
ResponsesWebSocketTokenUsageProcessor,
|
||||
batch_cost_calculator,
|
||||
completion_cost,
|
||||
cost_per_token,
|
||||
handle_realtime_stream_cost_calculation,
|
||||
response_cost_calculator,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_model_response_object,
|
||||
|
|
@ -23,11 +34,25 @@ from litellm.llms.together_ai.cost_calculator import get_model_params_and_catego
|
|||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
Choices,
|
||||
CompletionTokensDetailsWrapper,
|
||||
EmbeddingResponse,
|
||||
ImageObject,
|
||||
ImageResponse,
|
||||
ImageUsage,
|
||||
ImageUsageInputTokensDetails,
|
||||
LiteLLMRealtimeStreamLoggingObject,
|
||||
Message,
|
||||
ModelInfo,
|
||||
ModelResponse,
|
||||
PromptTokensDetails,
|
||||
PromptTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import supports_prompt_caching
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -7801,7 +7826,7 @@ def test_completion_cost_model_response_cost(response_model, custom_llm_provider
|
|||
|
||||
def test_select_model_name_for_cost_calc():
|
||||
from litellm.cost_calculator import select_model_name_for_cost_calc
|
||||
from litellm.types.utils import ModelResponse, Choices, Usage, Message
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
args = {
|
||||
"model": "Mistral-large-nmefg",
|
||||
|
|
|
|||
|
|
@ -3249,7 +3249,9 @@ class TestExtraBodyCannotOverrideModel:
|
|||
}
|
||||
]
|
||||
|
||||
untouched = litellm.get_optional_params(model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools())
|
||||
untouched = litellm.get_optional_params(
|
||||
model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools()
|
||||
)
|
||||
assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched
|
||||
|
||||
result = litellm.get_optional_params(
|
||||
|
|
@ -4240,9 +4242,7 @@ def test_is_prompt_caching_valid_prompt_stops_counting_once_the_minimum_is_reach
|
|||
assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=1024) is True
|
||||
assert sum(counted_messages) < len(long_prompt), sum(counted_messages)
|
||||
|
||||
full_count = litellm.token_counter(
|
||||
model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True
|
||||
)
|
||||
full_count = litellm.token_counter(model="claude-opus-4-8", messages=long_prompt, use_default_image_token_count=True)
|
||||
assert (
|
||||
is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=long_prompt, min_token_count=full_count)
|
||||
is True
|
||||
|
|
@ -4709,13 +4709,7 @@ _SUCCESS_RESPONSES_BY_CALL_TYPE: Final = (
|
|||
pytest.param(EmbeddingResponse(model="text-embedding-3-small"), CallTypes.aembedding, id="embedding"),
|
||||
pytest.param(
|
||||
ResponsesAPIResponse(
|
||||
id="resp_abc",
|
||||
created_at=1,
|
||||
output=[],
|
||||
parallel_tool_calls=False,
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
model="gpt-5.6",
|
||||
id="resp_abc", created_at=1, output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], model="gpt-5.6"
|
||||
),
|
||||
CallTypes.aresponses,
|
||||
id="responses",
|
||||
|
|
@ -4743,9 +4737,7 @@ async def test_success_deployment_hook_raising_keeps_response_and_runs_later_hoo
|
|||
|
||||
assert result is response
|
||||
assert second_hook.seen_responses == (response,)
|
||||
failure_logs: Final = tuple(
|
||||
r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message
|
||||
)
|
||||
failure_logs: Final = tuple(r for r in caplog.records if "async_post_call_success_deployment_hook error" in r.message)
|
||||
assert len(failure_logs) == 1
|
||||
assert "_ChatShapedSuccessDeploymentHook" in failure_logs[0].message
|
||||
assert str(call_type) in failure_logs[0].message
|
||||
|
|
@ -4882,13 +4874,17 @@ async def test_wrapper_async_logs_converted_responses_stream_with_standard_loggi
|
|||
({"metadata": None}, True, False),
|
||||
],
|
||||
)
|
||||
def test_is_litellm_router_call_is_async_aware(kwargs: Mapping[str, object], is_async: bool, expected: bool) -> None:
|
||||
def test_is_litellm_router_call_is_async_aware(
|
||||
kwargs: Mapping[str, object], is_async: bool, expected: bool
|
||||
) -> None:
|
||||
assert _is_litellm_router_call(kwargs, is_async=is_async) is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_streaming", "streaming"])
|
||||
async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
async def test_router_aresponses_does_not_run_sdk_retries(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
model_list: Final = [
|
||||
|
|
@ -4902,7 +4898,9 @@ async def test_router_aresponses_does_not_run_sdk_retries(monkeypatch: pytest.Mo
|
|||
},
|
||||
}
|
||||
]
|
||||
router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True
|
||||
)
|
||||
|
||||
try:
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
|
|
@ -4938,7 +4936,9 @@ def test_router_responses_keeps_sdk_retries_for_sync_router_call(
|
|||
},
|
||||
}
|
||||
]
|
||||
router: Final = litellm.Router(model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=model_list, num_retries=0, retry_after=0, disable_cooldowns=True
|
||||
)
|
||||
|
||||
try:
|
||||
with respx.mock(assert_all_called=True) as respx_mock:
|
||||
|
|
@ -6782,9 +6782,7 @@ def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summa
|
|||
|
||||
|
||||
@pytest.mark.parametrize("original_function", ("atext_completion", "text_completion"))
|
||||
def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(
|
||||
original_function: str,
|
||||
) -> None:
|
||||
def test_function_setup_without_a_prompt_leaves_the_missing_prompt_to_request_validation(original_function: str) -> None:
|
||||
assert _logged_request_messages(original_function, model="gpt-4o") is None
|
||||
|
||||
|
||||
|
|
@ -6826,7 +6824,6 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
|
|
@ -6840,7 +6837,6 @@ def setup_and_teardown():
|
|||
loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
|
||||
|
||||
MODEL: Final = "anthropic/claude-haiku-4-5"
|
||||
|
||||
|
||||
|
|
@ -6850,7 +6846,6 @@ def test_validate_tool_choice_none():
|
|||
result = validate_chat_completion_tool_choice(None, model=MODEL)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_string():
|
||||
"""Test that string values are returned as-is."""
|
||||
|
|
@ -6858,7 +6853,6 @@ def test_validate_tool_choice_string():
|
|||
assert validate_chat_completion_tool_choice("none", model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice("required", model=MODEL) == "required"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_standard_dict():
|
||||
"""Test standard OpenAI format with function."""
|
||||
|
|
@ -6866,7 +6860,6 @@ def test_validate_tool_choice_standard_dict():
|
|||
result = validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert result == tool_choice
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_cursor_format():
|
||||
"""Cursor IDE format {"type": "auto"} is unwrapped to the bare string."""
|
||||
|
|
@ -6874,7 +6867,6 @@ def test_validate_tool_choice_cursor_format():
|
|||
assert validate_chat_completion_tool_choice({"type": "none"}, model=MODEL) == "none"
|
||||
assert validate_chat_completion_tool_choice({"type": "required"}, model=MODEL) == "required"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
|
|
@ -6896,7 +6888,6 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice):
|
|||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == MODEL
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
@pytest.mark.parametrize("tool_choice", [123, []])
|
||||
def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
|
||||
|
|
@ -6907,7 +6898,6 @@ def test_validate_tool_choice_invalid_type_is_a_400(tool_choice):
|
|||
validate_chat_completion_tool_choice(tool_choice, model=MODEL)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
def test_validate_tool_choice_without_model_is_still_a_400():
|
||||
"""Callers that predate the model argument keep getting a 400, with an empty model on the error."""
|
||||
|
|
@ -6916,14 +6906,12 @@ def test_validate_tool_choice_without_model_is_still_a_400():
|
|||
assert exc_info.value.status_code == 400
|
||||
assert exc_info.value.model == ""
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def _vcr_outcome_gate_local_testing(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def isolate_litellm_state():
|
||||
"""
|
||||
|
|
@ -6976,7 +6964,6 @@ def isolate_litellm_state():
|
|||
setattr(litellm, attr, original_value)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
_SCALAR_DEFAULTS = {
|
||||
"num_retries": getattr(litellm, "num_retries", None),
|
||||
"num_retries_per_request": getattr(litellm, "num_retries_per_request", None),
|
||||
|
|
@ -7020,7 +7007,6 @@ def setup_and_teardown_local_testing():
|
|||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7046,7 +7032,6 @@ def test_vertex_projects():
|
|||
assert "vertex_ai_project" in optional_params
|
||||
assert "vertex_ai_location" in optional_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7062,7 +7047,6 @@ def test_bedrock_embed_v2_regular():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params == {"dimensions": 512}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7081,7 +7065,6 @@ def test_bedrock_embed_v2_with_drop_params():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params == {"dimensions": 512, "embeddingTypes": ["binary"]}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7103,7 +7086,6 @@ def test_openai_non_text_embedding_3_with_allowed_openai_params():
|
|||
print(f"received optional_params: {optional_params}")
|
||||
assert optional_params.get("dimensions") == 1024
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7130,7 +7112,6 @@ def test_openai_non_text_embedding_3_without_allowed_openai_params_raises():
|
|||
finally:
|
||||
litellm.drop_params = prev_drop_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7159,7 +7140,6 @@ def test_openai_non_text_embedding_3_drop_params_per_call():
|
|||
finally:
|
||||
litellm.drop_params = prev_drop_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
"_vcr_outcome_gate_local_testing",
|
||||
"isolate_litellm_state",
|
||||
|
|
@ -7187,14 +7167,12 @@ def test_openai_non_text_embedding_3_drop_params_global():
|
|||
finally:
|
||||
litellm.drop_params = prev_drop_params
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def _vcr_outcome_gate_search_tests(request, vcr):
|
||||
install_live_call_probe(request, vcr)
|
||||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_search_tests")
|
||||
def test_search_tool_name_in_all_litellm_params():
|
||||
"""
|
||||
|
|
@ -7204,7 +7182,6 @@ def test_search_tool_name_in_all_litellm_params():
|
|||
"""
|
||||
assert "search_tool_name" in all_litellm_params
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate_search_tests")
|
||||
def test_filter_out_search_tool_name():
|
||||
"""
|
||||
|
|
@ -7231,7 +7208,6 @@ def test_filter_out_search_tool_name():
|
|||
assert filtered["query"] == "latest ai developments"
|
||||
assert filtered["max_results"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Chat over the Responses bridge exits two @client wrappers with one logging object. Issue
|
||||
|
|
@ -7384,7 +7360,9 @@ def test_trimming_with_system_message_within_max_tokens():
|
|||
"content": "This is a medium normal message, let's say litellm is awesome.",
|
||||
},
|
||||
]
|
||||
trimmed_messages = trim_messages(messages, max_tokens=30, model="gpt-4-0613")
|
||||
trimmed_messages = trim_messages(
|
||||
messages, max_tokens=30, model="gpt-4-0613"
|
||||
)
|
||||
assert len(trimmed_messages) == 2
|
||||
assert trimmed_messages[0]["content"] == "This is a short system message"
|
||||
|
||||
|
|
@ -7567,7 +7545,9 @@ def test_aget_valid_models():
|
|||
print(valid_models)
|
||||
|
||||
# list of openai supported llms on litellm
|
||||
expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
|
||||
expected_models = (
|
||||
litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
|
||||
)
|
||||
|
||||
assert set(valid_models) == set(expected_models)
|
||||
|
||||
|
|
@ -7656,7 +7636,9 @@ def test_get_chat_completion_prompt():
|
|||
prompt_variables=None,
|
||||
)
|
||||
|
||||
assert litellm_logging_obj.messages == [{"role": "user", "content": updated_message}]
|
||||
assert litellm_logging_obj.messages == [
|
||||
{"role": "user", "content": updated_message}
|
||||
]
|
||||
|
||||
|
||||
def test_redact_msgs_from_logs():
|
||||
|
|
@ -7728,7 +7710,9 @@ def test_redact_embedding_response():
|
|||
litellm.turn_off_message_logging = True
|
||||
|
||||
# Create a test EmbeddingResponse with usage data
|
||||
original_usage = litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10)
|
||||
original_usage = litellm.Usage(
|
||||
prompt_tokens=10, completion_tokens=0, total_tokens=10
|
||||
)
|
||||
original_data = [
|
||||
{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3, 0.4, 0.5]},
|
||||
{"object": "embedding", "index": 1, "embedding": [0.6, 0.7, 0.8, 0.9, 1.0]},
|
||||
|
|
@ -7764,7 +7748,9 @@ def test_redact_embedding_response():
|
|||
|
||||
# Assert the redacted response preserves critical metadata
|
||||
assert _redacted_response_obj.usage == original_usage # usage should be preserved
|
||||
assert _redacted_response_obj.model == "text-embedding-3-small" # model should be preserved
|
||||
assert (
|
||||
_redacted_response_obj.model == "text-embedding-3-small"
|
||||
) # model should be preserved
|
||||
assert _redacted_response_obj.object == "list" # object should be preserved
|
||||
|
||||
# Assert sensitive data is cleared
|
||||
|
|
@ -7818,8 +7804,12 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
)
|
||||
|
||||
# Test Case 1: standard_callback_dynamic_params = False (or not set)
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=False)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(
|
||||
turn_off_message_logging=False
|
||||
)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -7828,8 +7818,12 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
assert _redacted_response_obj.choices[0].message.content == test_content
|
||||
|
||||
# Test Case 2: standard_callback_dynamic_params = True
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(turn_off_message_logging=True)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams(
|
||||
turn_off_message_logging=True
|
||||
)
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -7840,7 +7834,9 @@ def test_redact_msgs_from_logs_with_dynamic_params():
|
|||
# Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging
|
||||
# since litellm.turn_off_message_logging is True redaction should occur
|
||||
standard_callback_dynamic_params = StandardCallbackDynamicParams()
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = standard_callback_dynamic_params
|
||||
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (
|
||||
standard_callback_dynamic_params
|
||||
)
|
||||
_redacted_response_obj = redact_message_input_output_from_logging(
|
||||
result=response_obj,
|
||||
model_call_details=litellm_logging_obj.model_call_details,
|
||||
|
|
@ -8184,7 +8180,10 @@ def test_models_by_provider():
|
|||
continue
|
||||
elif k == "sample_spec":
|
||||
continue
|
||||
elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse":
|
||||
elif (
|
||||
v["litellm_provider"] == "sagemaker"
|
||||
or v["litellm_provider"] == "bedrock_converse"
|
||||
):
|
||||
continue
|
||||
elif v.get("mode") in ("search", "decisions", "evaluation"):
|
||||
continue
|
||||
|
|
@ -8192,7 +8191,9 @@ def test_models_by_provider():
|
|||
providers.add(v["litellm_provider"])
|
||||
|
||||
for provider in providers:
|
||||
assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(provider)
|
||||
assert provider in models_by_provider.keys() or JSONProviderRegistry.exists(
|
||||
provider
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -8214,11 +8215,16 @@ def restore_end_user_cost_tracking_flags(monkeypatch: pytest.MonkeyPatch) -> Non
|
|||
({"user_api_key_end_user_id": "123"}, True, None),
|
||||
],
|
||||
)
|
||||
def test_get_end_user_id_for_cost_tracking(litellm_params, disable_end_user_cost_tracking, expected_end_user_id):
|
||||
def test_get_end_user_id_for_cost_tracking(
|
||||
litellm_params, disable_end_user_cost_tracking, expected_end_user_id
|
||||
):
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
litellm.disable_end_user_cost_tracking = disable_end_user_cost_tracking
|
||||
assert get_end_user_id_for_cost_tracking(litellm_params=litellm_params) == expected_end_user_id
|
||||
assert (
|
||||
get_end_user_id_for_cost_tracking(litellm_params=litellm_params)
|
||||
== expected_end_user_id
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_end_user_cost_tracking_flags")
|
||||
|
|
@ -8235,9 +8241,13 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
):
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
litellm.enable_end_user_cost_tracking_prometheus_only = enable_end_user_cost_tracking_prometheus_only
|
||||
litellm.enable_end_user_cost_tracking_prometheus_only = (
|
||||
enable_end_user_cost_tracking_prometheus_only
|
||||
)
|
||||
assert (
|
||||
get_end_user_id_for_cost_tracking(litellm_params=litellm_params, service_type="prometheus")
|
||||
get_end_user_id_for_cost_tracking(
|
||||
litellm_params=litellm_params, service_type="prometheus"
|
||||
)
|
||||
== expected_end_user_id
|
||||
)
|
||||
|
||||
|
|
@ -8252,14 +8262,20 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
),
|
||||
# Test with only litellm_metadata field (new behavior)
|
||||
(
|
||||
{"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"}},
|
||||
{
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
}
|
||||
},
|
||||
"user_from_litellm_metadata",
|
||||
),
|
||||
# Test with both fields - metadata should take precedence for user_api_key fields
|
||||
(
|
||||
{
|
||||
"metadata": {"user_api_key_end_user_id": "user_from_metadata"},
|
||||
"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
},
|
||||
},
|
||||
"user_from_metadata",
|
||||
),
|
||||
|
|
@ -8275,7 +8291,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
(
|
||||
{
|
||||
"metadata": {},
|
||||
"litellm_metadata": {"user_api_key_end_user_id": "user_from_litellm_metadata"},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_end_user_id": "user_from_litellm_metadata"
|
||||
},
|
||||
},
|
||||
"user_from_litellm_metadata",
|
||||
),
|
||||
|
|
@ -8283,7 +8301,9 @@ def test_get_end_user_id_for_cost_tracking_prometheus_only(
|
|||
({}, None),
|
||||
],
|
||||
)
|
||||
def test_get_end_user_id_for_cost_tracking_metadata_handling(litellm_params, expected_end_user_id):
|
||||
def test_get_end_user_id_for_cost_tracking_metadata_handling(
|
||||
litellm_params, expected_end_user_id
|
||||
):
|
||||
"""
|
||||
Test that get_end_user_id_for_cost_tracking correctly handles both metadata and litellm_metadata
|
||||
fields using the get_litellm_metadata_from_kwargs helper function.
|
||||
|
|
@ -8542,7 +8562,9 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch):
|
|||
assert len(litellm.success_callback) == curr_len_success_callback
|
||||
assert len(litellm.failure_callback) == curr_len_failure_callback
|
||||
|
||||
assert any(isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback)
|
||||
assert any(
|
||||
isinstance(callback, OpenMeterLogger) for callback in litellm.failure_callback
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -8639,14 +8661,20 @@ def test_validate_user_messages_invalid_content_type():
|
|||
[
|
||||
{
|
||||
"name": "default_on_guardrail",
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=True)],
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=True)
|
||||
],
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": []}}},
|
||||
"expected": ["test_guardrail"],
|
||||
},
|
||||
{
|
||||
"name": "request_specific_guardrail",
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)],
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}},
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=False)
|
||||
],
|
||||
"kwargs": {
|
||||
"metadata": {"requester_metadata": {"guardrails": ["test_guardrail"]}}
|
||||
},
|
||||
"expected": ["test_guardrail"],
|
||||
},
|
||||
{
|
||||
|
|
@ -8655,12 +8683,18 @@ def test_validate_user_messages_invalid_content_type():
|
|||
CustomGuardrail(guardrail_name="default_guardrail", default_on=True),
|
||||
CustomGuardrail(guardrail_name="request_guardrail", default_on=False),
|
||||
],
|
||||
"kwargs": {"metadata": {"requester_metadata": {"guardrails": ["request_guardrail"]}}},
|
||||
"kwargs": {
|
||||
"metadata": {
|
||||
"requester_metadata": {"guardrails": ["request_guardrail"]}
|
||||
}
|
||||
},
|
||||
"expected": ["default_guardrail", "request_guardrail"],
|
||||
},
|
||||
{
|
||||
"name": "empty_metadata",
|
||||
"callbacks": [CustomGuardrail(guardrail_name="test_guardrail", default_on=False)],
|
||||
"callbacks": [
|
||||
CustomGuardrail(guardrail_name="test_guardrail", default_on=False)
|
||||
],
|
||||
"kwargs": {},
|
||||
"expected": [],
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue