mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(otel): type the pass-through operation tests and drive them with a real logging object at fixed times
This commit is contained in:
parent
c7b3d77056
commit
ccf84e6006
6 changed files with 36 additions and 27 deletions
|
|
@ -24,6 +24,7 @@ dashboard already querying that histogram.
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -504,15 +505,12 @@ def test_agent_message_is_not_labelled_as_chat(call_type):
|
|||
assert dp.attributes[OPERATION_KEY] == "invoke_agent"
|
||||
|
||||
|
||||
def test_passthrough_operation_labels_the_metrics():
|
||||
reader = InMemoryMetricReader()
|
||||
def test_passthrough_operation_labels_the_metrics() -> None:
|
||||
reader: Final = InMemoryMetricReader()
|
||||
kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint")
|
||||
labelled: Final = {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
|
||||
|
||||
asyncio.run(
|
||||
_logger(reader, enable_metrics=True).async_log_success_event(
|
||||
{**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}, response_obj, start, end
|
||||
)
|
||||
)
|
||||
asyncio.run(_logger(reader, enable_metrics=True).async_log_success_event(labelled, response_obj, start, end))
|
||||
|
||||
for dp in _metrics_by_name(reader)[OPERATION_DURATION]:
|
||||
assert dp.attributes[OPERATION_KEY] == "retrieval"
|
||||
|
|
|
|||
|
|
@ -32,13 +32,13 @@ from litellm.integrations.otel import (
|
|||
from litellm.integrations.otel.mappers.genai import GenAIMapper
|
||||
from litellm.integrations.otel.model import spans as spans_mod
|
||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
EmbeddingOutput,
|
||||
LLMCallSpanData,
|
||||
RequestIdentity,
|
||||
_upstream_address_port,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
|
||||
from litellm.integrations.otel.model.spans import (
|
||||
SPAN_REGISTRY,
|
||||
LiteLLMSpanKind,
|
||||
|
|
@ -224,10 +224,10 @@ def test_operation_resolution():
|
|||
assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL
|
||||
|
||||
|
||||
def test_passthrough_operation_wins_over_the_call_type():
|
||||
def test_passthrough_operation_wins_over_the_call_type() -> None:
|
||||
assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL
|
||||
assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT
|
||||
event = LLMCallEvent.from_dict(
|
||||
event: Final = LLMCallEvent.from_dict(
|
||||
{"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
|
||||
)
|
||||
assert event.operation is GenAIOperation.RETRIEVAL
|
||||
|
|
|
|||
|
|
@ -1049,9 +1049,9 @@ class TestOpenTelemetrySemconvStability(unittest.TestCase):
|
|||
kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"}
|
||||
self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5")
|
||||
|
||||
def test_passthrough_operation_names_the_span(self):
|
||||
h = self._make(env="gen_ai_latest_experimental")
|
||||
kwargs = {
|
||||
def test_passthrough_operation_names_the_span(self) -> None:
|
||||
h: Final = self._make(env="gen_ai_latest_experimental")
|
||||
kwargs: Final = {
|
||||
"model": "unknown",
|
||||
"call_type": "pass_through_endpoint",
|
||||
PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -122,9 +122,18 @@ def test_streaming_generate_content_bills_against_the_requested_provider(
|
|||
)
|
||||
def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
|
||||
call_type: str, expected: GenAIOperation | None
|
||||
):
|
||||
logging_obj = _logging_obj()
|
||||
logging_obj.model_call_details["call_type"] = call_type
|
||||
) -> None:
|
||||
start: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[],
|
||||
stream=True,
|
||||
call_type=call_type,
|
||||
start_time=start,
|
||||
litellm_call_id="search-1",
|
||||
function_id="search",
|
||||
)
|
||||
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
|
||||
|
||||
PassThroughStreamingHandler._build_passthrough_logging_result(
|
||||
litellm_logging_obj=logging_obj,
|
||||
|
|
@ -132,9 +141,9 @@ def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
|
|||
url_route="https://doc-app.internal/search",
|
||||
request_body={},
|
||||
endpoint_type=EndpointType.GENERIC,
|
||||
start_time=datetime.now(),
|
||||
start_time=start,
|
||||
raw_bytes=[b'data: {"chunk": 1}\n\n'],
|
||||
end_time=datetime.now(),
|
||||
end_time=datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc),
|
||||
model=None,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -659,14 +659,14 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response(
|
|||
)
|
||||
async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it(
|
||||
reported: GenAIOperation | None, expected: GenAIOperation
|
||||
):
|
||||
) -> None:
|
||||
url: Final = "https://doc-app.internal/search"
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
start_time=_START,
|
||||
litellm_call_id="search-1",
|
||||
function_id="search",
|
||||
)
|
||||
|
|
@ -679,8 +679,8 @@ async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_nam
|
|||
logging_obj=logging_obj,
|
||||
url_route=url,
|
||||
result="",
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
start_time=_START,
|
||||
end_time=_END,
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
|
@ -131,8 +133,8 @@ def test_apply_only_overwrites_what_upstream_reported():
|
|||
|
||||
|
||||
@pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS])
|
||||
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation):
|
||||
logging_obj = _logging_obj()
|
||||
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation) -> None:
|
||||
logging_obj: Final = _logging_obj()
|
||||
|
||||
apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value}))
|
||||
|
||||
|
|
@ -143,8 +145,8 @@ def test_apply_operation_records_the_operation_the_upstream_named(operation: Gen
|
|||
"headers",
|
||||
[{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}],
|
||||
)
|
||||
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]):
|
||||
logging_obj = _logging_obj()
|
||||
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]) -> None:
|
||||
logging_obj: Final = _logging_obj()
|
||||
|
||||
apply_upstream_reported_operation(logging_obj, _headers(**headers))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue