mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
596c51a5fa
* feat(telemetry): add AggregatingSink, fixed-bucket histograms, Report and HttpExporter * fix(telemetry): add records to rows in place, guard concurrent flushes, cap restored rows and count bad histogram values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(telemetry): give the shared row helper its lint reasons Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(telemetry): retry the same statuses as the proxy's should_retry (408, 409, 429, 5xx) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(telemetry): cover populated block_types and header_keys in report JSON Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: nate <nate@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
83 lines
3.5 KiB
Python
83 lines
3.5 KiB
Python
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from litellm.telemetry.consent import OFF, REQUIRES, TelemetryConsent
|
|
from litellm.telemetry.records import TelemetryGroup
|
|
from litellm.telemetry.report import report_to_json
|
|
from litellm.telemetry.sample import sample_report
|
|
|
|
_REQUEST_FIELDS: Final = {
|
|
TelemetryGroup.REQUEST_SUCCESS: {
|
|
"endpoint",
|
|
"litellm_status",
|
|
"handled_by_rust",
|
|
"latency_to_first_byte_ms",
|
|
"request_count",
|
|
},
|
|
TelemetryGroup.TOKEN_INFO: {"provider_cache_hit", "input_tokens", "cache_read_tokens"},
|
|
TelemetryGroup.REQUEST_TAXONOMY: {"provider", "deployment_hash"},
|
|
TelemetryGroup.EVENT_DETAILS: {"block_count", "block_types", "header_keys"},
|
|
}
|
|
|
|
|
|
def _with_requirements(group: TelemetryGroup) -> frozenset[TelemetryGroup]:
|
|
required: Final = REQUIRES[group]
|
|
parents: Final[frozenset[TelemetryGroup]] = _with_requirements(required) if required is not None else frozenset()
|
|
return frozenset({group}) | parents
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_off_builds_no_report() -> None:
|
|
assert await sample_report(OFF, litellm_version="1.0.0") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("group", list(_REQUEST_FIELDS))
|
|
async def test_each_request_group_adds_exactly_its_own_fields(group: TelemetryGroup) -> None:
|
|
groups: Final = _with_requirements(group)
|
|
with_group: Final = await sample_report(TelemetryConsent(groups), litellm_version="1.0.0")
|
|
without_group: Final = await sample_report(TelemetryConsent(groups - {group}), litellm_version="1.0.0")
|
|
assert with_group is not None and without_group is not None
|
|
rows_with: Final = report_to_json(with_group)["requests"]
|
|
assert isinstance(rows_with, list) and isinstance(rows_with[0], dict)
|
|
rows_without: Final = report_to_json(without_group).get("requests", [{}])
|
|
assert isinstance(rows_without, list) and isinstance(rows_without[0], dict)
|
|
assert _REQUEST_FIELDS[group] <= set(rows_with[0]) - set(rows_without[0])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heartbeat_only_sample_is_just_the_instance_header() -> None:
|
|
report: Final = await sample_report(TelemetryConsent(frozenset({TelemetryGroup.HEARTBEAT})), litellm_version="2.0")
|
|
assert report is not None
|
|
assert report_to_json(report) == {
|
|
"schema_version": 1,
|
|
"instance": {"instance_id": "0" * 32, "litellm_version": "2.0", "groups": ["heartbeat"]},
|
|
"window_start": 0.0,
|
|
"window_end": 300.0,
|
|
"dropped_records": 0,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_without_taxonomy_rows_from_different_providers_with_the_same_status_merge() -> None:
|
|
consent: Final = TelemetryConsent(_with_requirements(TelemetryGroup.TOKEN_INFO))
|
|
report: Final = await sample_report(consent, litellm_version="1.0.0")
|
|
assert report is not None
|
|
assert len(report.requests) == 2
|
|
assert all(key.provider is None for key, _ in report.requests)
|
|
assert report.attempts == ()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_instance_configuration_adds_config_keys_and_page_navigation_adds_ui_events() -> None:
|
|
consent: Final = TelemetryConsent(
|
|
frozenset({TelemetryGroup.HEARTBEAT, TelemetryGroup.INSTANCE_CONFIGURATION, TelemetryGroup.PAGE_NAVIGATION})
|
|
)
|
|
report: Final = await sample_report(consent, litellm_version="1.0.0")
|
|
assert report is not None
|
|
json_report: Final = report_to_json(report)
|
|
instance: Final = json_report["instance"]
|
|
assert isinstance(instance, dict) and instance["config_keys"]
|
|
assert json_report["ui_events"]
|
|
assert "requests" not in json_report
|