mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(otel): type the pass-through operation tests and drive them with a real logging object at fixed times
This commit is contained in:
parent
c7b3d77056
commit
ccf84e6006
6 changed files with 36 additions and 27 deletions
|
|
@ -24,6 +24,7 @@ dashboard already querying that histogram.
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
@ -504,15 +505,12 @@ def test_agent_message_is_not_labelled_as_chat(call_type):
|
||||||
assert dp.attributes[OPERATION_KEY] == "invoke_agent"
|
assert dp.attributes[OPERATION_KEY] == "invoke_agent"
|
||||||
|
|
||||||
|
|
||||||
def test_passthrough_operation_labels_the_metrics():
|
def test_passthrough_operation_labels_the_metrics() -> None:
|
||||||
reader = InMemoryMetricReader()
|
reader: Final = InMemoryMetricReader()
|
||||||
kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint")
|
kwargs, response_obj, start, end = _build_call(call_type="pass_through_endpoint")
|
||||||
|
labelled: Final = {**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
|
||||||
|
|
||||||
asyncio.run(
|
asyncio.run(_logger(reader, enable_metrics=True).async_log_success_event(labelled, response_obj, start, end))
|
||||||
_logger(reader, enable_metrics=True).async_log_success_event(
|
|
||||||
{**kwargs, PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}, response_obj, start, end
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
for dp in _metrics_by_name(reader)[OPERATION_DURATION]:
|
for dp in _metrics_by_name(reader)[OPERATION_DURATION]:
|
||||||
assert dp.attributes[OPERATION_KEY] == "retrieval"
|
assert dp.attributes[OPERATION_KEY] == "retrieval"
|
||||||
|
|
|
||||||
|
|
@ -32,13 +32,13 @@ from litellm.integrations.otel import (
|
||||||
from litellm.integrations.otel.mappers.genai import GenAIMapper
|
from litellm.integrations.otel.mappers.genai import GenAIMapper
|
||||||
from litellm.integrations.otel.model import spans as spans_mod
|
from litellm.integrations.otel.model import spans as spans_mod
|
||||||
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
from litellm.integrations.otel.model.metadata import LLMCallEvent
|
||||||
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
|
|
||||||
from litellm.integrations.otel.model.payloads import (
|
from litellm.integrations.otel.model.payloads import (
|
||||||
EmbeddingOutput,
|
EmbeddingOutput,
|
||||||
LLMCallSpanData,
|
LLMCallSpanData,
|
||||||
RequestIdentity,
|
RequestIdentity,
|
||||||
_upstream_address_port,
|
_upstream_address_port,
|
||||||
)
|
)
|
||||||
|
from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY
|
||||||
from litellm.integrations.otel.model.spans import (
|
from litellm.integrations.otel.model.spans import (
|
||||||
SPAN_REGISTRY,
|
SPAN_REGISTRY,
|
||||||
LiteLLMSpanKind,
|
LiteLLMSpanKind,
|
||||||
|
|
@ -224,10 +224,10 @@ def test_operation_resolution():
|
||||||
assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL
|
assert resolve_operation("call_mcp_tool") is GenAIOperation.EXECUTE_TOOL
|
||||||
|
|
||||||
|
|
||||||
def test_passthrough_operation_wins_over_the_call_type():
|
def test_passthrough_operation_wins_over_the_call_type() -> None:
|
||||||
assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL
|
assert resolve_operation("pass_through_endpoint", GenAIOperation.RETRIEVAL) is GenAIOperation.RETRIEVAL
|
||||||
assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT
|
assert resolve_operation("pass_through_endpoint", "retrieval") is GenAIOperation.CHAT
|
||||||
event = LLMCallEvent.from_dict(
|
event: Final = LLMCallEvent.from_dict(
|
||||||
{"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
|
{"call_type": "pass_through_endpoint", "model": "unknown", PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL}
|
||||||
)
|
)
|
||||||
assert event.operation is GenAIOperation.RETRIEVAL
|
assert event.operation is GenAIOperation.RETRIEVAL
|
||||||
|
|
|
||||||
|
|
@ -1049,9 +1049,9 @@ class TestOpenTelemetrySemconvStability(unittest.TestCase):
|
||||||
kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"}
|
kwargs = {"model": "claude-sonnet-4-5", "call_type": "unknown"}
|
||||||
self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5")
|
self.assertEqual(h._get_span_name(kwargs), "chat claude-sonnet-4-5")
|
||||||
|
|
||||||
def test_passthrough_operation_names_the_span(self):
|
def test_passthrough_operation_names_the_span(self) -> None:
|
||||||
h = self._make(env="gen_ai_latest_experimental")
|
h: Final = self._make(env="gen_ai_latest_experimental")
|
||||||
kwargs = {
|
kwargs: Final = {
|
||||||
"model": "unknown",
|
"model": "unknown",
|
||||||
"call_type": "pass_through_endpoint",
|
"call_type": "pass_through_endpoint",
|
||||||
PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL,
|
PASSTHROUGH_OPERATION_KEY: GenAIOperation.RETRIEVAL,
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from typing import Final
|
from typing import Final
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
|
@ -122,9 +122,18 @@ def test_streaming_generate_content_bills_against_the_requested_provider(
|
||||||
)
|
)
|
||||||
def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
|
def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
|
||||||
call_type: str, expected: GenAIOperation | None
|
call_type: str, expected: GenAIOperation | None
|
||||||
):
|
) -> None:
|
||||||
logging_obj = _logging_obj()
|
start: Final = datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone.utc)
|
||||||
logging_obj.model_call_details["call_type"] = call_type
|
logging_obj: Final = LiteLLMLoggingObj(
|
||||||
|
model="unknown",
|
||||||
|
messages=[],
|
||||||
|
stream=True,
|
||||||
|
call_type=call_type,
|
||||||
|
start_time=start,
|
||||||
|
litellm_call_id="search-1",
|
||||||
|
function_id="search",
|
||||||
|
)
|
||||||
|
logging_obj.update_environment_variables(litellm_params={}, optional_params={})
|
||||||
|
|
||||||
PassThroughStreamingHandler._build_passthrough_logging_result(
|
PassThroughStreamingHandler._build_passthrough_logging_result(
|
||||||
litellm_logging_obj=logging_obj,
|
litellm_logging_obj=logging_obj,
|
||||||
|
|
@ -132,9 +141,9 @@ def test_unparsed_stream_is_labelled_passthrough_only_for_a_passthrough_call(
|
||||||
url_route="https://doc-app.internal/search",
|
url_route="https://doc-app.internal/search",
|
||||||
request_body={},
|
request_body={},
|
||||||
endpoint_type=EndpointType.GENERIC,
|
endpoint_type=EndpointType.GENERIC,
|
||||||
start_time=datetime.now(),
|
start_time=start,
|
||||||
raw_bytes=[b'data: {"chunk": 1}\n\n'],
|
raw_bytes=[b'data: {"chunk": 1}\n\n'],
|
||||||
end_time=datetime.now(),
|
end_time=datetime(2025, 1, 2, 3, 4, 9, tzinfo=timezone.utc),
|
||||||
model=None,
|
model=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -659,14 +659,14 @@ def test_vertex_ai_live_route_without_usage_frames_yields_no_logging_response(
|
||||||
)
|
)
|
||||||
async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it(
|
async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_named_it(
|
||||||
reported: GenAIOperation | None, expected: GenAIOperation
|
reported: GenAIOperation | None, expected: GenAIOperation
|
||||||
):
|
) -> None:
|
||||||
url: Final = "https://doc-app.internal/search"
|
url: Final = "https://doc-app.internal/search"
|
||||||
logging_obj: Final = LiteLLMLoggingObj(
|
logging_obj: Final = LiteLLMLoggingObj(
|
||||||
model="unknown",
|
model="unknown",
|
||||||
messages=[],
|
messages=[],
|
||||||
stream=False,
|
stream=False,
|
||||||
call_type="pass_through_endpoint",
|
call_type="pass_through_endpoint",
|
||||||
start_time=datetime.now(),
|
start_time=_START,
|
||||||
litellm_call_id="search-1",
|
litellm_call_id="search-1",
|
||||||
function_id="search",
|
function_id="search",
|
||||||
)
|
)
|
||||||
|
|
@ -679,8 +679,8 @@ async def test_unparsed_response_is_labelled_passthrough_unless_the_upstream_nam
|
||||||
logging_obj=logging_obj,
|
logging_obj=logging_obj,
|
||||||
url_route=url,
|
url_route=url,
|
||||||
result="",
|
result="",
|
||||||
start_time=datetime.now(),
|
start_time=_START,
|
||||||
end_time=datetime.now(),
|
end_time=_END,
|
||||||
cache_hit=False,
|
cache_hit=False,
|
||||||
request_body={},
|
request_body={},
|
||||||
passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"),
|
passthrough_logging_payload=PassthroughStandardLoggingPayload(url=url, request_body={}, request_method="POST"),
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
|
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
@ -131,8 +133,8 @@ def test_apply_only_overwrites_what_upstream_reported():
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS])
|
@pytest.mark.parametrize("operation", [GenAIOperation.RETRIEVAL, GenAIOperation.EMBEDDINGS])
|
||||||
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation):
|
def test_apply_operation_records_the_operation_the_upstream_named(operation: GenAIOperation) -> None:
|
||||||
logging_obj = _logging_obj()
|
logging_obj: Final = _logging_obj()
|
||||||
|
|
||||||
apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value}))
|
apply_upstream_reported_operation(logging_obj, _headers(**{"x-litellm-operation-name": operation.value}))
|
||||||
|
|
||||||
|
|
@ -143,8 +145,8 @@ def test_apply_operation_records_the_operation_the_upstream_named(operation: Gen
|
||||||
"headers",
|
"headers",
|
||||||
[{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}],
|
[{}, {"x-litellm-operation-name": "summarize"}, {"x-litellm-operation-name": "Retrieval"}],
|
||||||
)
|
)
|
||||||
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]):
|
def test_apply_operation_ignores_a_missing_or_unknown_name(headers: dict[str, str]) -> None:
|
||||||
logging_obj = _logging_obj()
|
logging_obj: Final = _logging_obj()
|
||||||
|
|
||||||
apply_upstream_reported_operation(logging_obj, _headers(**headers))
|
apply_upstream_reported_operation(logging_obj, _headers(**headers))
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue